From f13862872770cad328434824d2050ccec0b3ddc4 Mon Sep 17 00:00:00 2001 From: potya Date: Wed, 20 May 2020 01:18:49 +0300 Subject: [PATCH 001/329] Add cast_keep_nullable setting --- src/Core/Settings.h | 1 + src/Functions/FunctionsConversion.h | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 61fcf658ba8..8697e155024 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -428,6 +428,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \ M(SettingBool, partial_merge_join, false, "Obsolete. Use join_algorithm='prefer_partial_merge' instead.", 0) \ M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ + M(SettingBool, cast_keep_nullable, true, "Cast operator keep Nullable for new data type", 0) \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 64708f45598..207e7759683 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -41,6 +41,7 @@ #include #include #include +#include namespace DB @@ -2406,6 +2407,11 @@ protected: " Instead there is a column with the following structure: " + column->dumpStructure(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + Settings set; + if (set.cast_keep_nullable) + if (arguments.back().type->isNullable()) { + return makeNullable(DataTypeFactory::instance().get(type_col->getValue())); + } return DataTypeFactory::instance().get(type_col->getValue()); } From 88709d0f01a88e66906fa8944ca433b3fa2f5fe3 Mon Sep 17 00:00:00 2001 From: potya Date: Wed, 20 May 2020 01:25:13 +0300 Subject: [PATCH 002/329] Back same as master --- src/DataTypes/DataTypeString.cpp | 40 +++------- src/DataTypes/DataTypesNumber.cpp | 128 +++--------------------------- 2 files changed, 22 insertions(+), 146 deletions(-) diff --git a/src/DataTypes/DataTypeString.cpp b/src/DataTypes/DataTypeString.cpp index 358afc6c8f2..cdeb6fe1012 100644 --- a/src/DataTypes/DataTypeString.cpp +++ b/src/DataTypes/DataTypeString.cpp @@ -15,9 +15,6 @@ #include #include -#include -#include - #include #include #include @@ -30,14 +27,6 @@ namespace DB { - -namespace ErrorCodes -{ - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int UNEXPECTED_AST_STRUCTURE; -} - - void DataTypeString::serializeBinary(const Field & field, WriteBuffer & ostr) const { const String & s = get(field); @@ -104,8 +93,8 @@ void DataTypeString::serializeBinaryBulk(const IColumn & column, WriteBuffer & o return; size_t end = limit && offset + limit < size - ? offset + limit - : size; + ? offset + limit + : size; if (offset == 0) { @@ -377,38 +366,29 @@ bool DataTypeString::equals(const IDataType & rhs) const return typeid(rhs) == typeid(*this); } -static DataTypePtr create(const ASTPtr & arguments) -{ - if (arguments) { - if (arguments->children.size() > 1) - throw Exception("String data type family mustnt have more than one argument - size in characters", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - const auto * argument = arguments->children[0]->as(); - if (!argument || argument->value.getType() != Field::Types::UInt64 || argument->value.get() == 0) - throw Exception("FixedString data type family may have only a number (positive integer) as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - } - - return std::make_shared(); -} - void registerDataTypeString(DataTypeFactory & factory) { - factory.registerDataType("String", create); + auto creator = static_cast([] { return DataTypePtr(std::make_shared()); }); + + factory.registerSimpleDataType("String", creator); /// These synonyms are added for compatibility. factory.registerAlias("CHAR", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("CHARACTER", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("VARCHAR", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("VARCHAR2", "String", DataTypeFactory::CaseInsensitive); /// Oracle factory.registerAlias("TEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("MEDIUMTEXT", "String", DataTypeFactory::CaseInsensitive); - factory.registerAlias("LONG", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("LONGTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("BLOB", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("CLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYBLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("MEDIUMBLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("LONGBLOB", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("BYTEA", "String", DataTypeFactory::CaseInsensitive); /// PostgreSQL } -} +} \ No newline at end of file diff --git a/src/DataTypes/DataTypesNumber.cpp b/src/DataTypes/DataTypesNumber.cpp index f260daa15b4..cd28b4fdad8 100644 --- a/src/DataTypes/DataTypesNumber.cpp +++ b/src/DataTypes/DataTypesNumber.cpp @@ -2,115 +2,9 @@ #include -#include -#include - - namespace DB { -namespace ErrorCodes -{ - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int UNEXPECTED_AST_STRUCTURE; -} - -static DataTypePtr createForInt8(const ASTPtr & arguments) -{ - if (arguments) { - if (arguments->children.size() > 1) - throw Exception("INT8 data type family must not have more than one argument - display width", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - const auto * argument = arguments->children[0]->as(); - if (!argument || argument->value.getType() != Field::Types::UInt64 || argument->value.get() == 0) - throw Exception("INT8 data type family may have only a number (positive integer) as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - } - - return std::make_shared(); -} - -static DataTypePtr createForInt16(const ASTPtr & arguments) -{ - if (arguments) { - if (arguments->children.size() > 1) - throw Exception("INT16 data type family must not have more than one argument - display width", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - const auto * argument = arguments->children[0]->as(); - if (!argument || argument->value.getType() != Field::Types::UInt64 || argument->value.get() == 0) - throw Exception("INT16 data type family may have only a number (positive integer) as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - } - - return std::make_shared(); -} - -static DataTypePtr createForInt32(const ASTPtr & arguments) -{ - if (arguments) { - if (arguments->children.size() > 1) - throw Exception("INT32 data type family must not have more than one argument - display width", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - const auto * argument = arguments->children[0]->as(); - if (!argument || argument->value.getType() != Field::Types::UInt64 || argument->value.get() == 0) - throw Exception("INT32 data type family may have only a number (positive integer) as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - } - - return std::make_shared(); -} - -static DataTypePtr createForInt64(const ASTPtr & arguments) -{ - if (arguments) { - if (arguments->children.size() > 1) - throw Exception("INT64 data type family must not have more than one argument - display width", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - const auto * argument = arguments->children[0]->as(); - if (!argument || argument->value.getType() != Field::Types::UInt64 || argument->value.get() == 0) - throw Exception("INT64 data type family may have only a number (positive integer) as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - } - - return std::make_shared(); -} - -static DataTypePtr createForFloat32(const ASTPtr & arguments) -{ - if (arguments) { - if (arguments->children.size() > 2) - throw Exception("FLOAT32 data type family must not have more than two arguments - total number of digits and number of digits following the decimal point", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - else if (arguments->children.size() == 1) { - const auto * argument = arguments->children[0]->as(); - if (!argument || argument->value.getType() != Field::Types::UInt64) - throw Exception("FLOAT32 data type family may have a non negative number as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - } else if (arguments->children.size() == 2) { - const auto * beforePoint = arguments->children[0]->as(); - const auto * afterPoint = arguments->children[1]->as(); - if (!beforePoint || beforePoint->value.getType() != Field::Types::UInt64 || - !afterPoint|| afterPoint->value.getType() != Field::Types::UInt64) - throw Exception("FLOAT32 data type family may have a non negative number as its arguments", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - } - } - - return std::make_shared(); -} - -static DataTypePtr createForFloat64(const ASTPtr & arguments) -{ - if (arguments) { - if (arguments->children.size() != 2) - throw Exception("FLOAT64 data type family must have only two arguments - total number of digits and number of digits following the decimal point", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - else { - const auto * beforePoint = arguments->children[0]->as(); - const auto * afterPoint = arguments->children[1]->as(); - if (!beforePoint || beforePoint->value.getType() != Field::Types::UInt64 || - !afterPoint|| afterPoint->value.getType() != Field::Types::UInt64) - throw Exception("FLOAT64 data type family may have a non negative number as its arguments", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - } - } - - return std::make_shared(); -} - - - void registerDataTypeNumbers(DataTypeFactory & factory) { factory.registerSimpleDataType("UInt8", [] { return DataTypePtr(std::make_shared()); }); @@ -118,27 +12,29 @@ void registerDataTypeNumbers(DataTypeFactory & factory) factory.registerSimpleDataType("UInt32", [] { return DataTypePtr(std::make_shared()); }); factory.registerSimpleDataType("UInt64", [] { return DataTypePtr(std::make_shared()); }); - factory.registerDataType("Int8", createForInt8); - factory.registerDataType("Int16", createForInt16); - factory.registerDataType("Int32", createForInt32); - factory.registerDataType("Int64", createForInt64); - factory.registerDataType("Float32", createForFloat32); - factory.registerDataType("Float64", createForFloat64); + factory.registerSimpleDataType("Int8", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("Int16", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("Int32", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("Int64", [] { return DataTypePtr(std::make_shared()); }); + + factory.registerSimpleDataType("Float32", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("Float64", [] { return DataTypePtr(std::make_shared()); }); /// These synonyms are added for compatibility. factory.registerAlias("TINYINT", "Int8", DataTypeFactory::CaseInsensitive); factory.registerAlias("BOOL", "Int8", DataTypeFactory::CaseInsensitive); factory.registerAlias("BOOLEAN", "Int8", DataTypeFactory::CaseInsensitive); - factory.registerAlias("INT1", "Int8", DataTypeFactory::CaseInsensitive); + factory.registerAlias("INT1", "Int8", DataTypeFactory::CaseInsensitive); /// MySQL + factory.registerAlias("BYTE", "Int8", DataTypeFactory::CaseInsensitive); /// MS Access factory.registerAlias("SMALLINT", "Int16", DataTypeFactory::CaseInsensitive); - factory.registerAlias("INT2", "Int16", DataTypeFactory::CaseInsensitive); factory.registerAlias("INT", "Int32", DataTypeFactory::CaseInsensitive); - factory.registerAlias("INT4", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("INTEGER", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("BIGINT", "Int64", DataTypeFactory::CaseInsensitive); factory.registerAlias("FLOAT", "Float32", DataTypeFactory::CaseInsensitive); + factory.registerAlias("REAL", "Float32", DataTypeFactory::CaseInsensitive); + factory.registerAlias("SINGLE", "Float32", DataTypeFactory::CaseInsensitive); /// MS Access factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::CaseInsensitive); } -} +} \ No newline at end of file From f3c1cdb69617231b0d85caf1b66293de68c6eeeb Mon Sep 17 00:00:00 2001 From: potya Date: Wed, 20 May 2020 01:41:54 +0300 Subject: [PATCH 003/329] one more back to master --- src/DataTypes/DataTypeString.cpp | 9 +++------ src/DataTypes/DataTypesNumber.cpp | 7 +++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/DataTypes/DataTypeString.cpp b/src/DataTypes/DataTypeString.cpp index cdeb6fe1012..d0db66b202b 100644 --- a/src/DataTypes/DataTypeString.cpp +++ b/src/DataTypes/DataTypeString.cpp @@ -93,8 +93,8 @@ void DataTypeString::serializeBinaryBulk(const IColumn & column, WriteBuffer & o return; size_t end = limit && offset + limit < size - ? offset + limit - : size; + ? offset + limit + : size; if (offset == 0) { @@ -376,19 +376,16 @@ void registerDataTypeString(DataTypeFactory & factory) /// These synonyms are added for compatibility. factory.registerAlias("CHAR", "String", DataTypeFactory::CaseInsensitive); - factory.registerAlias("CHARACTER", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("VARCHAR", "String", DataTypeFactory::CaseInsensitive); - factory.registerAlias("VARCHAR2", "String", DataTypeFactory::CaseInsensitive); /// Oracle factory.registerAlias("TEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("MEDIUMTEXT", "String", DataTypeFactory::CaseInsensitive); + factory.registerAlias("LONG", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("LONGTEXT", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("BLOB", "String", DataTypeFactory::CaseInsensitive); - factory.registerAlias("CLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("TINYBLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("MEDIUMBLOB", "String", DataTypeFactory::CaseInsensitive); factory.registerAlias("LONGBLOB", "String", DataTypeFactory::CaseInsensitive); - factory.registerAlias("BYTEA", "String", DataTypeFactory::CaseInsensitive); /// PostgreSQL } } \ No newline at end of file diff --git a/src/DataTypes/DataTypesNumber.cpp b/src/DataTypes/DataTypesNumber.cpp index cd28b4fdad8..82a1f35c297 100644 --- a/src/DataTypes/DataTypesNumber.cpp +++ b/src/DataTypes/DataTypesNumber.cpp @@ -25,15 +25,14 @@ void registerDataTypeNumbers(DataTypeFactory & factory) factory.registerAlias("TINYINT", "Int8", DataTypeFactory::CaseInsensitive); factory.registerAlias("BOOL", "Int8", DataTypeFactory::CaseInsensitive); factory.registerAlias("BOOLEAN", "Int8", DataTypeFactory::CaseInsensitive); - factory.registerAlias("INT1", "Int8", DataTypeFactory::CaseInsensitive); /// MySQL - factory.registerAlias("BYTE", "Int8", DataTypeFactory::CaseInsensitive); /// MS Access + factory.registerAlias("INT1", "Int8", DataTypeFactory::CaseInsensitive); factory.registerAlias("SMALLINT", "Int16", DataTypeFactory::CaseInsensitive); + factory.registerAlias("INT2", "Int16", DataTypeFactory::CaseInsensitive); factory.registerAlias("INT", "Int32", DataTypeFactory::CaseInsensitive); + factory.registerAlias("INT4", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("INTEGER", "Int32", DataTypeFactory::CaseInsensitive); factory.registerAlias("BIGINT", "Int64", DataTypeFactory::CaseInsensitive); factory.registerAlias("FLOAT", "Float32", DataTypeFactory::CaseInsensitive); - factory.registerAlias("REAL", "Float32", DataTypeFactory::CaseInsensitive); - factory.registerAlias("SINGLE", "Float32", DataTypeFactory::CaseInsensitive); /// MS Access factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::CaseInsensitive); } From cee9c1517e5621e00c1efba830d798f1877612b4 Mon Sep 17 00:00:00 2001 From: potya Date: Wed, 20 May 2020 01:51:06 +0300 Subject: [PATCH 004/329] cast_keep_nullable set to false --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 8697e155024..5e17e915cbe 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -428,7 +428,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \ M(SettingBool, partial_merge_join, false, "Obsolete. Use join_algorithm='prefer_partial_merge' instead.", 0) \ M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ - M(SettingBool, cast_keep_nullable, true, "Cast operator keep Nullable for new data type", 0) \ + M(SettingBool, cast_keep_nullable, false, "Cast operator keep Nullable for new data type", 0) \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) From d8bb98bbcf401e962ceb92351abc006f1ce330de Mon Sep 17 00:00:00 2001 From: potya Date: Wed, 20 May 2020 04:22:32 +0300 Subject: [PATCH 005/329] Add NULL and NOY NULL modifiers for data types in create query --- src/Core/Settings.h | 1 + src/Interpreters/InterpreterCreateQuery.cpp | 22 +++++++++++++++++++++ src/Parsers/ASTColumnDeclaration.cpp | 10 ++++++++++ src/Parsers/ASTColumnDeclaration.h | 2 ++ src/Parsers/ParserCreateQuery.h | 21 ++++++++++++++++++++ 5 files changed, 56 insertions(+) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5e17e915cbe..501cd61e70c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -429,6 +429,7 @@ struct Settings : public SettingsCollection M(SettingBool, partial_merge_join, false, "Obsolete. Use join_algorithm='prefer_partial_merge' instead.", 0) \ M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ M(SettingBool, cast_keep_nullable, false, "Cast operator keep Nullable for new data type", 0) \ + M(SettingBool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 20c9dc839f4..d4609477c1e 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -273,10 +274,13 @@ ASTPtr InterpreterCreateQuery::formatConstraints(const ConstraintsDescription & ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpressionList & columns_ast, const Context & context) { + Settings set; + /// First, deduce implicit types. /** all default_expressions as a single expression list, * mixed with conversion-columns for each explicitly specified type */ + ASTPtr default_expr_list = std::make_shared(); NamesAndTypesList column_names_and_types; @@ -285,9 +289,27 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres const auto & col_decl = ast->as(); DataTypePtr column_type = nullptr; + if (col_decl.isNULL && col_decl.isNotNULL) + throw Exception{"Cant use NOT NULL and NULL together", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + if (col_decl.type) { column_type = DataTypeFactory::instance().get(col_decl.type); + + if (col_decl.isNULL) { + if (column_type->isNullable()) + throw Exception{"Cant use NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + else { + column_type = makeNullable(column_type); + } + } else if (col_decl.isNotNULL) { + if (column_type->isNullable()) + throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + } + + if (set.data_type_default_nullable && !column_type->isNullable()) + column_type = makeNullable(column_type); + column_names_and_types.emplace_back(col_decl.name, column_type); } else diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index b281315f555..c43a4b554a4 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -16,6 +16,16 @@ ASTPtr ASTColumnDeclaration::clone() const res->children.push_back(res->type); } + if (isNULL) + { + res->isNULL = isNULL; + } + + if (isNULL) + { + res->isNotNULL = isNotNULL; + } + if (default_expression) { res->default_expression = default_expression->clone(); diff --git a/src/Parsers/ASTColumnDeclaration.h b/src/Parsers/ASTColumnDeclaration.h index ad23e0669bc..4816a433991 100644 --- a/src/Parsers/ASTColumnDeclaration.h +++ b/src/Parsers/ASTColumnDeclaration.h @@ -13,6 +13,8 @@ class ASTColumnDeclaration : public IAST public: String name; ASTPtr type; + bool isNULL; + bool isNotNULL; String default_specifier; ASTPtr default_expression; ASTPtr comment; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 19410a78dd2..7cf2497b3fb 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -10,6 +10,8 @@ #include #include +#include + namespace DB { @@ -115,6 +117,8 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E NameParser name_parser; ParserIdentifierWithOptionalParameters type_parser; ParserKeyword s_default{"DEFAULT"}; + ParserKeyword s_null{"NULL"}; + ParserKeyword s_not_null{"NOT NULL"}; ParserKeyword s_materialized{"MATERIALIZED"}; ParserKeyword s_alias{"ALIAS"}; ParserKeyword s_comment{"COMMENT"}; @@ -135,6 +139,8 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E */ ASTPtr type; String default_specifier; + bool isNull = false; + bool isNotNull = false; ASTPtr default_expression; ASTPtr comment_expression; ASTPtr codec_expression; @@ -163,6 +169,13 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (require_type && !type && !default_expression) return false; /// reject column name without type + if (s_null.checkWithoutMoving(pos, expected)) { + if (s_null.check(pos, expected)) + isNull = true; + } else if (s_not_null.checkWithoutMoving(pos, expected)) { + if (s_not_null.check(pos, expected)) + isNotNull = true; + } if (s_comment.ignore(pos, expected)) { @@ -193,6 +206,14 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E column_declaration->children.push_back(std::move(type)); } + if (isNull) { + column_declaration->isNULL = isNull; + } + + if (isNull) { + column_declaration->isNotNULL = isNotNull; + } + if (default_expression) { column_declaration->default_specifier = default_specifier; From a21cc0e76e488bb6414b80614a2621bbdfb10c71 Mon Sep 17 00:00:00 2001 From: potya Date: Wed, 20 May 2020 04:26:19 +0300 Subject: [PATCH 006/329] Add NULL and NOT NULL modifiers for data types in create query --- src/Core/Settings.h | 1 - src/Functions/FunctionsConversion.h | 6 ------ 2 files changed, 7 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 501cd61e70c..d0e2f459e8e 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -428,7 +428,6 @@ struct Settings : public SettingsCollection M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \ M(SettingBool, partial_merge_join, false, "Obsolete. Use join_algorithm='prefer_partial_merge' instead.", 0) \ M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ - M(SettingBool, cast_keep_nullable, false, "Cast operator keep Nullable for new data type", 0) \ M(SettingBool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \ diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 207e7759683..64708f45598 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -41,7 +41,6 @@ #include #include #include -#include namespace DB @@ -2407,11 +2406,6 @@ protected: " Instead there is a column with the following structure: " + column->dumpStructure(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - Settings set; - if (set.cast_keep_nullable) - if (arguments.back().type->isNullable()) { - return makeNullable(DataTypeFactory::instance().get(type_col->getValue())); - } return DataTypeFactory::instance().get(type_col->getValue()); } From 8a9064cef2b82fdb214359bb784f0c697919b810 Mon Sep 17 00:00:00 2001 From: potya Date: Wed, 20 May 2020 21:23:35 +0300 Subject: [PATCH 007/329] at start --- src/Interpreters/InterpreterCreateQuery.cpp | 4 +--- src/Parsers/ASTColumnDeclaration.cpp | 14 ++++++++++++ src/Parsers/ASTColumnDeclaration.h | 4 ++-- src/Parsers/ParserCreateQuery.h | 24 +++++++++++++-------- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d4609477c1e..ec83c3f9c7f 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -274,8 +274,6 @@ ASTPtr InterpreterCreateQuery::formatConstraints(const ConstraintsDescription & ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpressionList & columns_ast, const Context & context) { - Settings set; - /// First, deduce implicit types. /** all default_expressions as a single expression list, @@ -307,7 +305,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; } - if (set.data_type_default_nullable && !column_type->isNullable()) + if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.isNotNULL) column_type = makeNullable(column_type); column_names_and_types.emplace_back(col_decl.name, column_type); diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index c43a4b554a4..a6c4b819fdf 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -19,11 +19,13 @@ ASTPtr ASTColumnDeclaration::clone() const if (isNULL) { res->isNULL = isNULL; + res->children.push_back(res->isNULL); } if (isNULL) { res->isNotNULL = isNotNULL; + res->children.push_back(res->isNotNULL); } if (default_expression) @@ -69,6 +71,18 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta type->formatImpl(settings, state, frame); } + if (isNULL) + { + settings.ostr << ' '; + isNULL->formatImpl(settings, state, frame); + } + + if (isNotNULL) + { + settings.ostr << ' '; + isNotNULL->formatImpl(settings, state, frame); + } + if (default_expression) { settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << default_specifier << (settings.hilite ? hilite_none : "") << ' '; diff --git a/src/Parsers/ASTColumnDeclaration.h b/src/Parsers/ASTColumnDeclaration.h index 4816a433991..d3c50d453d5 100644 --- a/src/Parsers/ASTColumnDeclaration.h +++ b/src/Parsers/ASTColumnDeclaration.h @@ -13,8 +13,8 @@ class ASTColumnDeclaration : public IAST public: String name; ASTPtr type; - bool isNULL; - bool isNotNULL; + ASTPtr isNULL; + ASTPtr isNotNULL; String default_specifier; ASTPtr default_expression; ASTPtr comment; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 7cf2497b3fb..66a2005334d 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -128,6 +128,8 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ParserStringLiteral string_literal_parser; ParserCodec codec_parser; ParserExpression expression_parser; + ParserIdentifier null_parser; + ParserIdentifier not_null_parser; /// mandatory column name ASTPtr name; @@ -139,8 +141,8 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E */ ASTPtr type; String default_specifier; - bool isNull = false; - bool isNotNull = false; + ASTPtr isNull; + ASTPtr isNotNull; ASTPtr default_expression; ASTPtr comment_expression; ASTPtr codec_expression; @@ -169,12 +171,14 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (require_type && !type && !default_expression) return false; /// reject column name without type - if (s_null.checkWithoutMoving(pos, expected)) { - if (s_null.check(pos, expected)) - isNull = true; - } else if (s_not_null.checkWithoutMoving(pos, expected)) { - if (s_not_null.check(pos, expected)) - isNotNull = true; + Pos pos_before_null = pos; + + if (s_null.check(pos, expected)) { + if (!null_parser.parse(pos_before_null, isNull, expected)) + return false; + } else if (s_not_null.check(pos, expected)) { + if (!not_null_parser.parse(pos_before_null, isNotNull, expected)) + return false; } if (s_comment.ignore(pos, expected)) @@ -208,10 +212,12 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (isNull) { column_declaration->isNULL = isNull; + column_declaration->children.push_back(std::move(isNull)); } - if (isNull) { + if (isNotNull) { column_declaration->isNotNULL = isNotNull; + column_declaration->children.push_back(std::move(isNotNull)); } if (default_expression) From 7e5de33e93b350a5551900e633c2f6e983180a1e Mon Sep 17 00:00:00 2001 From: tavplubix Date: Fri, 22 May 2020 17:35:56 +0300 Subject: [PATCH 008/329] trigger CI --- src/DataTypes/DataTypeString.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataTypes/DataTypeString.cpp b/src/DataTypes/DataTypeString.cpp index d0db66b202b..c1afa8b90ea 100644 --- a/src/DataTypes/DataTypeString.cpp +++ b/src/DataTypes/DataTypeString.cpp @@ -388,4 +388,4 @@ void registerDataTypeString(DataTypeFactory & factory) factory.registerAlias("LONGBLOB", "String", DataTypeFactory::CaseInsensitive); } -} \ No newline at end of file +} From bee14177cd29dfda02e5c8d9543e85bae3b2b685 Mon Sep 17 00:00:00 2001 From: potya Date: Sat, 23 May 2020 17:32:47 +0300 Subject: [PATCH 009/329] Fix NOT nULL modifier --- src/Interpreters/InterpreterCreateQuery.cpp | 14 +++++----- src/Parsers/ASTColumnDeclaration.cpp | 18 +++++++------ src/Parsers/ASTColumnDeclaration.h | 2 +- src/Parsers/ParserCreateQuery.h | 29 ++++++++++++--------- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index ec83c3f9c7f..586f2d0f056 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -287,25 +287,25 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres const auto & col_decl = ast->as(); DataTypePtr column_type = nullptr; - if (col_decl.isNULL && col_decl.isNotNULL) - throw Exception{"Cant use NOT NULL and NULL together", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + if (!col_decl.isNULL && col_decl.isNot) + throw Exception{"Cant use NOT without NULL", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; if (col_decl.type) { column_type = DataTypeFactory::instance().get(col_decl.type); - if (col_decl.isNULL) { + if (col_decl.isNot && col_decl.isNULL) { + if (column_type->isNullable()) + throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + } else if (col_decl.isNULL && !col_decl.isNot) { if (column_type->isNullable()) throw Exception{"Cant use NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; else { column_type = makeNullable(column_type); } - } else if (col_decl.isNotNULL) { - if (column_type->isNullable()) - throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; } - if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.isNotNULL) + if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.isNot && !col_decl.isNULL) column_type = makeNullable(column_type); column_names_and_types.emplace_back(col_decl.name, column_type); diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index a6c4b819fdf..40513b45586 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -22,10 +22,10 @@ ASTPtr ASTColumnDeclaration::clone() const res->children.push_back(res->isNULL); } - if (isNULL) + if (isNot) { - res->isNotNULL = isNotNULL; - res->children.push_back(res->isNotNULL); + res->isNot = isNot; + res->children.push_back(res->isNot); } if (default_expression) @@ -71,17 +71,19 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta type->formatImpl(settings, state, frame); } + if (isNot) + { + settings.ostr << ' '; + isNot->formatImpl(settings, state, frame); + } + if (isNULL) { settings.ostr << ' '; isNULL->formatImpl(settings, state, frame); } - if (isNotNULL) - { - settings.ostr << ' '; - isNotNULL->formatImpl(settings, state, frame); - } + if (default_expression) { diff --git a/src/Parsers/ASTColumnDeclaration.h b/src/Parsers/ASTColumnDeclaration.h index d3c50d453d5..406a8cebded 100644 --- a/src/Parsers/ASTColumnDeclaration.h +++ b/src/Parsers/ASTColumnDeclaration.h @@ -14,7 +14,7 @@ public: String name; ASTPtr type; ASTPtr isNULL; - ASTPtr isNotNULL; + ASTPtr isNot; String default_specifier; ASTPtr default_expression; ASTPtr comment; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 66a2005334d..8976de8f1bc 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -118,7 +118,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ParserIdentifierWithOptionalParameters type_parser; ParserKeyword s_default{"DEFAULT"}; ParserKeyword s_null{"NULL"}; - ParserKeyword s_not_null{"NOT NULL"}; + ParserKeyword s_not{"NOT"}; ParserKeyword s_materialized{"MATERIALIZED"}; ParserKeyword s_alias{"ALIAS"}; ParserKeyword s_comment{"COMMENT"}; @@ -129,7 +129,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ParserCodec codec_parser; ParserExpression expression_parser; ParserIdentifier null_parser; - ParserIdentifier not_null_parser; + ParserCompoundIdentifier not_null_parser; /// mandatory column name ASTPtr name; @@ -142,7 +142,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ASTPtr type; String default_specifier; ASTPtr isNull; - ASTPtr isNotNull; + ASTPtr isNot; ASTPtr default_expression; ASTPtr comment_expression; ASTPtr codec_expression; @@ -171,14 +171,19 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (require_type && !type && !default_expression) return false; /// reject column name without type - Pos pos_before_null = pos; + // Pos pos_before_null = pos; - if (s_null.check(pos, expected)) { - if (!null_parser.parse(pos_before_null, isNull, expected)) - return false; - } else if (s_not_null.check(pos, expected)) { - if (!not_null_parser.parse(pos_before_null, isNotNull, expected)) + if (s_not.check(pos, expected)) { + if (s_null.check(pos, expected)) { + isNot = std::make_shared("NOT"); + isNull = std::make_shared("NULL"); + } else { return false; + } + } else { + if (s_null.check(pos, expected)) { + isNull = std::make_shared("NULL"); + } } if (s_comment.ignore(pos, expected)) @@ -215,9 +220,9 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E column_declaration->children.push_back(std::move(isNull)); } - if (isNotNull) { - column_declaration->isNotNULL = isNotNull; - column_declaration->children.push_back(std::move(isNotNull)); + if (isNot) { + column_declaration->isNot = isNot; + column_declaration->children.push_back(std::move(isNot)); } if (default_expression) From fb08a96582222ac6881efd2a9dd2d7edc7ec8c16 Mon Sep 17 00:00:00 2001 From: potya Date: Mon, 25 May 2020 14:11:51 +0300 Subject: [PATCH 010/329] Add simple test --- .../0_stateless/01269_creare_with_null.reference | 1 + tests/queries/0_stateless/01269_create_with_null.sql | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/queries/0_stateless/01269_creare_with_null.reference create mode 100644 tests/queries/0_stateless/01269_create_with_null.sql diff --git a/tests/queries/0_stateless/01269_creare_with_null.reference b/tests/queries/0_stateless/01269_creare_with_null.reference new file mode 100644 index 00000000000..fa7b52d9ebf --- /dev/null +++ b/tests/queries/0_stateless/01269_creare_with_null.reference @@ -0,0 +1 @@ +Nullable(Int32) Int32 Nullable(Int32) \ No newline at end of file diff --git a/tests/queries/0_stateless/01269_create_with_null.sql b/tests/queries/0_stateless/01269_create_with_null.sql new file mode 100644 index 00000000000..745fc9767a7 --- /dev/null +++ b/tests/queries/0_stateless/01269_create_with_null.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS data_null; + +CREATE TABLE data_null ( + a INT NULL, + b INT NOT NULL, + C Nullable(INT) +); + +INSERT INTO data_null VALUES (1, 2, 3); + +SELECT toTypeName(*) FROM data_null; \ No newline at end of file From 1ae82df3c0561b0f5968b919e124926212b02310 Mon Sep 17 00:00:00 2001 From: potya Date: Mon, 25 May 2020 14:20:33 +0300 Subject: [PATCH 011/329] at start --- src/Interpreters/InterpreterCreateQuery.cpp | 15 +++++++------ src/Parsers/ASTColumnDeclaration.cpp | 20 ++++++++--------- src/Parsers/ASTColumnDeclaration.h | 4 ++-- src/Parsers/ParserCreateQuery.h | 24 ++++++++++----------- 4 files changed, 31 insertions(+), 32 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 586f2d0f056..4c220d43b17 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -71,6 +71,7 @@ namespace ErrorCodes extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE; extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY; extern const int DICTIONARY_ALREADY_EXISTS; + extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE } @@ -287,25 +288,25 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres const auto & col_decl = ast->as(); DataTypePtr column_type = nullptr; - if (!col_decl.isNULL && col_decl.isNot) - throw Exception{"Cant use NOT without NULL", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + if (!col_decl.is_null && col_decl.is_not) + throw Exception{"Cant use NOT without NULL", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; if (col_decl.type) { column_type = DataTypeFactory::instance().get(col_decl.type); - if (col_decl.isNot && col_decl.isNULL) { + if (col_decl.is_not && col_decl.is_null) { if (column_type->isNullable()) - throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; - } else if (col_decl.isNULL && !col_decl.isNot) { + throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; + } else if (col_decl.is_null && !col_decl.is_not) { if (column_type->isNullable()) - throw Exception{"Cant use NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + throw Exception{"Cant use NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; else { column_type = makeNullable(column_type); } } - if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.isNot && !col_decl.isNULL) + if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.is_not && !col_decl.is_null) column_type = makeNullable(column_type); column_names_and_types.emplace_back(col_decl.name, column_type); diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index 40513b45586..de5abe28ffb 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -16,16 +16,16 @@ ASTPtr ASTColumnDeclaration::clone() const res->children.push_back(res->type); } - if (isNULL) + if (is_null) { - res->isNULL = isNULL; - res->children.push_back(res->isNULL); + res->is_null = is_null; + res->children.push_back(res->is_null); } - if (isNot) + if (is_not) { - res->isNot = isNot; - res->children.push_back(res->isNot); + res->is_not = is_not; + res->children.push_back(res->is_not); } if (default_expression) @@ -71,16 +71,16 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta type->formatImpl(settings, state, frame); } - if (isNot) + if (is_not) { settings.ostr << ' '; - isNot->formatImpl(settings, state, frame); + is_not->formatImpl(settings, state, frame); } - if (isNULL) + if (is_null) { settings.ostr << ' '; - isNULL->formatImpl(settings, state, frame); + is_null->formatImpl(settings, state, frame); } diff --git a/src/Parsers/ASTColumnDeclaration.h b/src/Parsers/ASTColumnDeclaration.h index 406a8cebded..34afd771de2 100644 --- a/src/Parsers/ASTColumnDeclaration.h +++ b/src/Parsers/ASTColumnDeclaration.h @@ -13,8 +13,8 @@ class ASTColumnDeclaration : public IAST public: String name; ASTPtr type; - ASTPtr isNULL; - ASTPtr isNot; + ASTPtr is_null; + ASTPtr is_not; String default_specifier; ASTPtr default_expression; ASTPtr comment; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 8976de8f1bc..c2b36460397 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -10,8 +10,6 @@ #include #include -#include - namespace DB { @@ -141,8 +139,8 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E */ ASTPtr type; String default_specifier; - ASTPtr isNull; - ASTPtr isNot; + ASTPtr is_null; + ASTPtr is_not; ASTPtr default_expression; ASTPtr comment_expression; ASTPtr codec_expression; @@ -175,14 +173,14 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (s_not.check(pos, expected)) { if (s_null.check(pos, expected)) { - isNot = std::make_shared("NOT"); - isNull = std::make_shared("NULL"); + is_not = std::make_shared("NOT"); + is_null = std::make_shared("NULL"); } else { return false; } } else { if (s_null.check(pos, expected)) { - isNull = std::make_shared("NULL"); + is_null = std::make_shared("NULL"); } } @@ -215,14 +213,14 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E column_declaration->children.push_back(std::move(type)); } - if (isNull) { - column_declaration->isNULL = isNull; - column_declaration->children.push_back(std::move(isNull)); + if (is_null) { + column_declaration->is_null = is_null; + column_declaration->children.push_back(std::move(is_null)); } - if (isNot) { - column_declaration->isNot = isNot; - column_declaration->children.push_back(std::move(isNot)); + if (is_not) { + column_declaration->is_not = is_not; + column_declaration->children.push_back(std::move(is_not)); } if (default_expression) From 70fac9c068f352dbe3a7ce96c81adc718770b409 Mon Sep 17 00:00:00 2001 From: potya Date: Mon, 25 May 2020 14:20:33 +0300 Subject: [PATCH 012/329] Fix problems --- src/Interpreters/InterpreterCreateQuery.cpp | 15 +++++++------ src/Parsers/ASTColumnDeclaration.cpp | 20 ++++++++--------- src/Parsers/ASTColumnDeclaration.h | 4 ++-- src/Parsers/ParserCreateQuery.h | 24 ++++++++++----------- 4 files changed, 31 insertions(+), 32 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 586f2d0f056..4c220d43b17 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -71,6 +71,7 @@ namespace ErrorCodes extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE; extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY; extern const int DICTIONARY_ALREADY_EXISTS; + extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE } @@ -287,25 +288,25 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres const auto & col_decl = ast->as(); DataTypePtr column_type = nullptr; - if (!col_decl.isNULL && col_decl.isNot) - throw Exception{"Cant use NOT without NULL", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + if (!col_decl.is_null && col_decl.is_not) + throw Exception{"Cant use NOT without NULL", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; if (col_decl.type) { column_type = DataTypeFactory::instance().get(col_decl.type); - if (col_decl.isNot && col_decl.isNULL) { + if (col_decl.is_not && col_decl.is_null) { if (column_type->isNullable()) - throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; - } else if (col_decl.isNULL && !col_decl.isNot) { + throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; + } else if (col_decl.is_null && !col_decl.is_not) { if (column_type->isNullable()) - throw Exception{"Cant use NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + throw Exception{"Cant use NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; else { column_type = makeNullable(column_type); } } - if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.isNot && !col_decl.isNULL) + if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.is_not && !col_decl.is_null) column_type = makeNullable(column_type); column_names_and_types.emplace_back(col_decl.name, column_type); diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index 40513b45586..de5abe28ffb 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -16,16 +16,16 @@ ASTPtr ASTColumnDeclaration::clone() const res->children.push_back(res->type); } - if (isNULL) + if (is_null) { - res->isNULL = isNULL; - res->children.push_back(res->isNULL); + res->is_null = is_null; + res->children.push_back(res->is_null); } - if (isNot) + if (is_not) { - res->isNot = isNot; - res->children.push_back(res->isNot); + res->is_not = is_not; + res->children.push_back(res->is_not); } if (default_expression) @@ -71,16 +71,16 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta type->formatImpl(settings, state, frame); } - if (isNot) + if (is_not) { settings.ostr << ' '; - isNot->formatImpl(settings, state, frame); + is_not->formatImpl(settings, state, frame); } - if (isNULL) + if (is_null) { settings.ostr << ' '; - isNULL->formatImpl(settings, state, frame); + is_null->formatImpl(settings, state, frame); } diff --git a/src/Parsers/ASTColumnDeclaration.h b/src/Parsers/ASTColumnDeclaration.h index 406a8cebded..34afd771de2 100644 --- a/src/Parsers/ASTColumnDeclaration.h +++ b/src/Parsers/ASTColumnDeclaration.h @@ -13,8 +13,8 @@ class ASTColumnDeclaration : public IAST public: String name; ASTPtr type; - ASTPtr isNULL; - ASTPtr isNot; + ASTPtr is_null; + ASTPtr is_not; String default_specifier; ASTPtr default_expression; ASTPtr comment; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 8976de8f1bc..c2b36460397 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -10,8 +10,6 @@ #include #include -#include - namespace DB { @@ -141,8 +139,8 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E */ ASTPtr type; String default_specifier; - ASTPtr isNull; - ASTPtr isNot; + ASTPtr is_null; + ASTPtr is_not; ASTPtr default_expression; ASTPtr comment_expression; ASTPtr codec_expression; @@ -175,14 +173,14 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (s_not.check(pos, expected)) { if (s_null.check(pos, expected)) { - isNot = std::make_shared("NOT"); - isNull = std::make_shared("NULL"); + is_not = std::make_shared("NOT"); + is_null = std::make_shared("NULL"); } else { return false; } } else { if (s_null.check(pos, expected)) { - isNull = std::make_shared("NULL"); + is_null = std::make_shared("NULL"); } } @@ -215,14 +213,14 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E column_declaration->children.push_back(std::move(type)); } - if (isNull) { - column_declaration->isNULL = isNull; - column_declaration->children.push_back(std::move(isNull)); + if (is_null) { + column_declaration->is_null = is_null; + column_declaration->children.push_back(std::move(is_null)); } - if (isNot) { - column_declaration->isNot = isNot; - column_declaration->children.push_back(std::move(isNot)); + if (is_not) { + column_declaration->is_not = is_not; + column_declaration->children.push_back(std::move(is_not)); } if (default_expression) From 2974d81b2ecb947d81973c65da9f3c9b46216087 Mon Sep 17 00:00:00 2001 From: potya Date: Mon, 25 May 2020 21:58:30 +0300 Subject: [PATCH 013/329] Fix erros and add test --- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- .../01269_create_with_null.reference | 2 + .../0_stateless/01269_create_with_null.sql | 39 +++++++++++++++++-- 3 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/01269_create_with_null.reference diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 4c220d43b17..81936f3706f 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -71,7 +71,7 @@ namespace ErrorCodes extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE; extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY; extern const int DICTIONARY_ALREADY_EXISTS; - extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE + extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE; } diff --git a/tests/queries/0_stateless/01269_create_with_null.reference b/tests/queries/0_stateless/01269_create_with_null.reference new file mode 100644 index 00000000000..7ef113393d5 --- /dev/null +++ b/tests/queries/0_stateless/01269_create_with_null.reference @@ -0,0 +1,2 @@ +Nullable(Int32) Int32 Nullable(Int32) Int32 +Nullable(Int32) Int32 Nullable(Int32) Nullable(Int32) diff --git a/tests/queries/0_stateless/01269_create_with_null.sql b/tests/queries/0_stateless/01269_create_with_null.sql index 745fc9767a7..68fa130e0da 100644 --- a/tests/queries/0_stateless/01269_create_with_null.sql +++ b/tests/queries/0_stateless/01269_create_with_null.sql @@ -1,11 +1,42 @@ DROP TABLE IF EXISTS data_null; +DROP TABLE IF EXISTS set_null; CREATE TABLE data_null ( a INT NULL, b INT NOT NULL, - C Nullable(INT) -); + c Nullable(INT), + d INT +) engine=Memory(); -INSERT INTO data_null VALUES (1, 2, 3); -SELECT toTypeName(*) FROM data_null; \ No newline at end of file +INSERT INTO data_null VALUES (1, 2, 3, 4); + +SELECT toTypeName(a), toTypeName(b), toTypeName(c), toTypeName(d) FROM data_null; + + +CREATE TABLE data_null ( + a Nullable(INT) NULL, + b INT NOT NULL, + c Nullable(INT) +) engine=Memory(); --{serverError 377} + + +CREATE TABLE data_null ( + a INT NULL, + b Nullable(INT) NOT NULL, + c Nullable(INT) +) engine=Memory(); --{serverError 377} + +SET data_type_default_nullable='true'; + +CREATE TABLE set_null ( + a INT NULL, + b INT NOT NULL, + c Nullable(INT), + d INT +) engine=Memory(); + + +INSERT INTO set_null VALUES (1, 2, 3, 4); + +SELECT toTypeName(a), toTypeName(b), toTypeName(c), toTypeName(d) FROM set_null; From 1fbcdbb58a315c56931ce4daf3499538a89b8622 Mon Sep 17 00:00:00 2001 From: potya Date: Mon, 25 May 2020 22:09:14 +0300 Subject: [PATCH 014/329] Fix error --- src/Interpreters/InterpreterCreateQuery.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index f92a751d64b..d4773d007e5 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -71,7 +71,7 @@ namespace ErrorCodes extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE; extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY; extern const int DICTIONARY_ALREADY_EXISTS; - extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE; + extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE; } @@ -288,25 +288,25 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres const auto & col_decl = ast->as(); DataTypePtr column_type = nullptr; - if (!col_decl.isNULL && col_decl.isNot) - throw Exception{"Cant use NOT without NULL", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + if (!col_decl.is_null && col_decl.is_not) + throw Exception{"Cant use NOT without NULL", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; if (col_decl.type) { column_type = DataTypeFactory::instance().get(col_decl.type); - if (col_decl.isNot && col_decl.isNULL) { + if (col_decl.is_not && col_decl.is_null) { if (column_type->isNullable()) - throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; - } else if (col_decl.isNULL && !col_decl.isNot) { + throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; + } else if (col_decl.is_null && !col_decl.is_not) { if (column_type->isNullable()) - throw Exception{"Cant use NULL with Nullable", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; + throw Exception{"Cant use NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; else { column_type = makeNullable(column_type); } } - if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.isNot && !col_decl.isNULL) + if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.is_not && !col_decl.is_null) column_type = makeNullable(column_type); column_names_and_types.emplace_back(col_decl.name, column_type); From d5840688a1b08a2de562f7cddad964b571a7105c Mon Sep 17 00:00:00 2001 From: potya Date: Wed, 27 May 2020 15:22:12 +0300 Subject: [PATCH 015/329] Fix style errors --- src/DataTypes/DataTypesNumber.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 7 ++++--- src/Parsers/ASTColumnDeclaration.cpp | 2 -- src/Parsers/ParserCreateQuery.h | 21 +++++++++++---------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/DataTypes/DataTypesNumber.cpp b/src/DataTypes/DataTypesNumber.cpp index 82a1f35c297..4da767ae359 100644 --- a/src/DataTypes/DataTypesNumber.cpp +++ b/src/DataTypes/DataTypesNumber.cpp @@ -36,4 +36,4 @@ void registerDataTypeNumbers(DataTypeFactory & factory) factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::CaseInsensitive); } -} \ No newline at end of file +} diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d4773d007e5..06a011e4633 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -298,12 +298,13 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres if (col_decl.is_not && col_decl.is_null) { if (column_type->isNullable()) throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; - } else if (col_decl.is_null && !col_decl.is_not) { + } + else if (col_decl.is_null && !col_decl.is_not) + { if (column_type->isNullable()) throw Exception{"Cant use NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; - else { + else column_type = makeNullable(column_type); - } } if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.is_not && !col_decl.is_null) diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index de5abe28ffb..2d5bcba611c 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -83,8 +83,6 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta is_null->formatImpl(settings, state, frame); } - - if (default_expression) { settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << default_specifier << (settings.hilite ? hilite_none : "") << ' '; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index c2b36460397..9fae3d60836 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -171,18 +171,17 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E // Pos pos_before_null = pos; - if (s_not.check(pos, expected)) { - if (s_null.check(pos, expected)) { + if (s_not.check(pos, expected)) + if (s_null.check(pos, expected)) + { is_not = std::make_shared("NOT"); is_null = std::make_shared("NULL"); - } else { + } + else return false; - } - } else { - if (s_null.check(pos, expected)) { + else + if (s_null.check(pos, expected)) is_null = std::make_shared("NULL"); - } - } if (s_comment.ignore(pos, expected)) { @@ -213,12 +212,14 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E column_declaration->children.push_back(std::move(type)); } - if (is_null) { + if (is_null) + { column_declaration->is_null = is_null; column_declaration->children.push_back(std::move(is_null)); } - if (is_not) { + if (is_not) + { column_declaration->is_not = is_not; column_declaration->children.push_back(std::move(is_not)); } From df13694a033cb07e44ca4bb94533a9d21b724f3e Mon Sep 17 00:00:00 2001 From: potya Date: Wed, 27 May 2020 15:32:39 +0300 Subject: [PATCH 016/329] Fix style errors --- src/Interpreters/InterpreterCreateQuery.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 06a011e4633..1b6f7d53a54 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -295,7 +295,8 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres { column_type = DataTypeFactory::instance().get(col_decl.type); - if (col_decl.is_not && col_decl.is_null) { + if (col_decl.is_not && col_decl.is_null) + { if (column_type->isNullable()) throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; } From 57df571e60b8f1a7cb0a0141a2129bedcbf3fae8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 1 Jun 2020 01:40:41 +0300 Subject: [PATCH 017/329] Remove trailing whitespaces from formatted queries in some cases --- src/Parsers/ASTExpressionList.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Parsers/ASTExpressionList.cpp b/src/Parsers/ASTExpressionList.cpp index 4f0b2d4cd6b..1395d8b15fe 100644 --- a/src/Parsers/ASTExpressionList.cpp +++ b/src/Parsers/ASTExpressionList.cpp @@ -37,10 +37,8 @@ void ASTExpressionList::formatImplMultiline(const FormatSettings & settings, For { if (separator) settings.ostr << separator; - settings.ostr << ' '; } - if (children.size() > 1) settings.ostr << indent_str; From 6eb6d8f3fd8beacbcb8ea536366e6c53ea833ff5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 2 Jun 2020 00:11:08 +0300 Subject: [PATCH 018/329] Get rid of annoying trailing whitespaces in CREATE query --- src/DataTypes/DataTypeTuple.cpp | 1 + src/Parsers/ASTColumnDeclaration.cpp | 3 -- src/Parsers/ASTConstraintDeclaration.cpp | 4 --- src/Parsers/ASTCreateQuery.cpp | 26 ++++++++--------- src/Parsers/ASTIndexDeclaration.cpp | 35 ++++++++++++++++++++++ src/Parsers/ASTIndexDeclaration.h | 37 ++---------------------- src/Parsers/ASTNameTypePair.cpp | 35 ++++++++++++++++++++++ src/Parsers/ASTNameTypePair.h | 24 ++------------- src/Parsers/ya.make | 2 ++ 9 files changed, 89 insertions(+), 78 deletions(-) create mode 100644 src/Parsers/ASTIndexDeclaration.cpp create mode 100644 src/Parsers/ASTNameTypePair.cpp diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp index 29db2a49b99..b69c4c31ca4 100644 --- a/src/DataTypes/DataTypeTuple.cpp +++ b/src/DataTypes/DataTypeTuple.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index b281315f555..15bf1d59574 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -47,9 +47,6 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta { frame.need_parens = false; - if (!settings.one_line) - settings.ostr << settings.nl_or_ws << std::string(4 * frame.indent, ' '); - /// We have to always backquote column names to avoid ambiguouty with INDEX and other declarations in CREATE query. settings.ostr << backQuote(name); diff --git a/src/Parsers/ASTConstraintDeclaration.cpp b/src/Parsers/ASTConstraintDeclaration.cpp index f268141f619..371bfa40f54 100644 --- a/src/Parsers/ASTConstraintDeclaration.cpp +++ b/src/Parsers/ASTConstraintDeclaration.cpp @@ -19,10 +19,6 @@ ASTPtr ASTConstraintDeclaration::clone() const void ASTConstraintDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { - frame.need_parens = false; - std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); - - s.ostr << s.nl_or_ws << indent_str; s.ostr << backQuoteIfNeed(name); s.ostr << (s.hilite ? hilite_keyword : "") << " CHECK " << (s.hilite ? hilite_none : ""); expr->formatImpl(s, state, frame); diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index d5942753f78..f7481ac3c09 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -108,17 +108,9 @@ void ASTColumnsElement::formatImpl(const FormatSettings & s, FormatState & state return; } - frame.need_parens = false; - std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); - - s.ostr << s.nl_or_ws << indent_str; s.ostr << (s.hilite ? hilite_keyword : "") << prefix << (s.hilite ? hilite_none : ""); - - FormatSettings nested_settings = s; - nested_settings.one_line = true; - nested_settings.nl_or_ws = ' '; - - elem->formatImpl(nested_settings, state, frame); + s.ostr << ' '; + elem->formatImpl(s, state, frame); } @@ -172,7 +164,12 @@ void ASTColumns::formatImpl(const FormatSettings & s, FormatState & state, Forma } if (!list.children.empty()) - list.formatImpl(s, state, frame); + { + if (s.one_line) + list.formatImpl(s, state, frame); + else + list.formatImplMultiline(s, state, frame); + } } @@ -277,7 +274,6 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat { settings.ostr << (settings.one_line ? " (" : "\n("); FormatStateStacked frame_nested = frame; - ++frame_nested.indent; columns_list->formatImpl(settings, state, frame_nested); settings.ostr << (settings.one_line ? ")" : "\n)"); } @@ -286,8 +282,10 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat { settings.ostr << (settings.one_line ? " (" : "\n("); FormatStateStacked frame_nested = frame; - ++frame_nested.indent; - dictionary_attributes_list->formatImpl(settings, state, frame_nested); + if (settings.one_line) + dictionary_attributes_list->formatImpl(settings, state, frame_nested); + else + dictionary_attributes_list->formatImplMultiline(settings, state, frame_nested); settings.ostr << (settings.one_line ? ")" : "\n)"); } diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp new file mode 100644 index 00000000000..e89f9bf26ed --- /dev/null +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -0,0 +1,35 @@ +#include +#include + + +namespace DB +{ + +ASTPtr ASTIndexDeclaration::clone() const +{ + auto res = std::make_shared(); + + res->name = name; + res->granularity = granularity; + + if (expr) + res->set(res->expr, expr->clone()); + if (type) + res->set(res->type, type->clone()); + return res; +} + + +void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const +{ + s.ostr << backQuoteIfNeed(name); + s.ostr << " "; + expr->formatImpl(s, state, frame); + s.ostr << (s.hilite ? hilite_keyword : "") << " TYPE " << (s.hilite ? hilite_none : ""); + type->formatImpl(s, state, frame); + s.ostr << (s.hilite ? hilite_keyword : "") << " GRANULARITY " << (s.hilite ? hilite_none : ""); + s.ostr << granularity; +} + +} + diff --git a/src/Parsers/ASTIndexDeclaration.h b/src/Parsers/ASTIndexDeclaration.h index c71ab21cf57..64ef6eb2db1 100644 --- a/src/Parsers/ASTIndexDeclaration.h +++ b/src/Parsers/ASTIndexDeclaration.h @@ -1,15 +1,8 @@ #pragma once -#include -#include -#include -#include -#include #include #include -#include - namespace DB { @@ -27,34 +20,8 @@ public: /** Get the text that identifies this element. */ String getID(char) const override { return "Index"; } - ASTPtr clone() const override - { - auto res = std::make_shared(); - - res->name = name; - res->granularity = granularity; - - if (expr) - res->set(res->expr, expr->clone()); - if (type) - res->set(res->type, type->clone()); - return res; - } - - void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override - { - frame.need_parens = false; - std::string indent_str = s.one_line ? "" : std::string(4 * frame.indent, ' '); - - s.ostr << s.nl_or_ws << indent_str; - s.ostr << backQuoteIfNeed(name); - s.ostr << " "; - expr->formatImpl(s, state, frame); - s.ostr << (s.hilite ? hilite_keyword : "") << " TYPE " << (s.hilite ? hilite_none : ""); - type->formatImpl(s, state, frame); - s.ostr << (s.hilite ? hilite_keyword : "") << " GRANULARITY " << (s.hilite ? hilite_none : ""); - s.ostr << granularity; - } + ASTPtr clone() const override; + void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; }; } diff --git a/src/Parsers/ASTNameTypePair.cpp b/src/Parsers/ASTNameTypePair.cpp new file mode 100644 index 00000000000..6c41d35315c --- /dev/null +++ b/src/Parsers/ASTNameTypePair.cpp @@ -0,0 +1,35 @@ +#include +#include + + +namespace DB +{ + +ASTPtr ASTNameTypePair::clone() const +{ + auto res = std::make_shared(*this); + res->children.clear(); + + if (type) + { + res->type = type; + res->children.push_back(res->type); + } + + return res; +} + + +void ASTNameTypePair::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); + + settings.ostr << '#'; + settings.ostr << indent_str << backQuoteIfNeed(name) << ' '; + type->formatImpl(settings, state, frame); + settings.ostr << '#'; +} + +} + + diff --git a/src/Parsers/ASTNameTypePair.h b/src/Parsers/ASTNameTypePair.h index 48dd7ae1ac9..638e980cbdc 100644 --- a/src/Parsers/ASTNameTypePair.h +++ b/src/Parsers/ASTNameTypePair.h @@ -1,7 +1,6 @@ #pragma once #include -#include namespace DB @@ -19,29 +18,10 @@ public: /** Get the text that identifies this element. */ String getID(char delim) const override { return "NameTypePair" + (delim + name); } - - ASTPtr clone() const override - { - auto res = std::make_shared(*this); - res->children.clear(); - - if (type) - { - res->type = type; - res->children.push_back(res->type); - } - - return res; - } + ASTPtr clone() const override; protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override - { - std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); - - settings.ostr << settings.nl_or_ws << indent_str << backQuoteIfNeed(name) << " "; - type->formatImpl(settings, state, frame); - } + void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index 8c7e4ff68af..c1cca094518 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -26,9 +26,11 @@ SRCS( ASTFunctionWithKeyValueArguments.cpp ASTGrantQuery.cpp ASTIdentifier.cpp + ASTIndexDeclaration.cpp ASTInsertQuery.cpp ASTKillQueryQuery.cpp ASTLiteral.cpp + ASTNameTypePair.cpp ASTOptimizeQuery.cpp ASTOrderByElement.cpp ASTPartition.cpp From 2a0da608fd34ad5d35e4ed14a6797451e548c718 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 2 Jun 2020 02:31:50 +0300 Subject: [PATCH 019/329] Update tests --- .../00597_push_down_predicate.reference | 50 +++++++++---------- ...51_default_databasename_for_view.reference | 8 +-- .../00826_cross_to_inner_join.reference | 16 +++--- .../00849_multiple_comma_join.reference | 24 ++++----- .../00849_multiple_comma_join_2.reference | 18 +++---- .../0_stateless/00908_analyze_query.reference | 2 +- ...0957_format_with_clashed_aliases.reference | 2 +- ...58_format_of_tuple_array_element.reference | 14 +++--- .../01056_predicate_optimizer_bugs.reference | 10 ++-- ...76_predicate_optimizer_with_view.reference | 8 +-- .../01083_cross_to_inner_with_like.reference | 6 +-- .../01278_format_multiple_queries.reference | 6 +-- 12 files changed, 82 insertions(+), 82 deletions(-) diff --git a/tests/queries/0_stateless/00597_push_down_predicate.reference b/tests/queries/0_stateless/00597_push_down_predicate.reference index 480b1c4525c..829c5a1577e 100644 --- a/tests/queries/0_stateless/00597_push_down_predicate.reference +++ b/tests/queries/0_stateless/00597_push_down_predicate.reference @@ -4,12 +4,12 @@ 1 2000-01-01 1 test string 1 1 -------Forbid push down------- -SELECT count()\nFROM \n(\n SELECT \n [number] AS a, \n [number * 2] AS b\n FROM system.numbers\n LIMIT 1\n) AS t\nARRAY JOIN \n a, \n b\nWHERE NOT ignore(a + b) +SELECT count()\nFROM \n(\n SELECT \n [number] AS a,\n [number * 2] AS b\n FROM system.numbers\n LIMIT 1\n) AS t\nARRAY JOIN \n a,\n b\nWHERE NOT ignore(a + b) 1 -SELECT \n a, \n b\nFROM \n(\n SELECT 1 AS a\n)\nANY LEFT JOIN \n(\n SELECT \n 1 AS a, \n 1 AS b\n) USING (a)\nWHERE b = 0 -SELECT \n a, \n b\nFROM \n(\n SELECT \n 1 AS a, \n 1 AS b\n)\nANY RIGHT JOIN \n(\n SELECT 1 AS a\n) USING (a)\nWHERE b = 0 -SELECT \n a, \n b\nFROM \n(\n SELECT 1 AS a\n)\nANY FULL OUTER JOIN \n(\n SELECT \n 1 AS a, \n 1 AS b\n) USING (a)\nWHERE b = 0 -SELECT \n a, \n b\nFROM \n(\n SELECT \n 1 AS a, \n 1 AS b\n)\nANY FULL OUTER JOIN \n(\n SELECT 1 AS a\n) USING (a)\nWHERE b = 0 +SELECT \n a,\n b\nFROM \n(\n SELECT 1 AS a\n)\nANY LEFT JOIN \n(\n SELECT \n 1 AS a,\n 1 AS b\n) USING (a)\nWHERE b = 0 +SELECT \n a,\n b\nFROM \n(\n SELECT \n 1 AS a,\n 1 AS b\n)\nANY RIGHT JOIN \n(\n SELECT 1 AS a\n) USING (a)\nWHERE b = 0 +SELECT \n a,\n b\nFROM \n(\n SELECT 1 AS a\n)\nANY FULL OUTER JOIN \n(\n SELECT \n 1 AS a,\n 1 AS b\n) USING (a)\nWHERE b = 0 +SELECT \n a,\n b\nFROM \n(\n SELECT \n 1 AS a,\n 1 AS b\n)\nANY FULL OUTER JOIN \n(\n SELECT 1 AS a\n) USING (a)\nWHERE b = 0 -------Need push down------- SELECT toString(value) AS value\nFROM \n(\n SELECT 1 AS value\n) 1 @@ -19,46 +19,46 @@ SELECT id\nFROM \n(\n SELECT arrayJoin([1, 2, 3]) AS id\n WHERE id = 1\n)\ 1 SELECT id\nFROM \n(\n SELECT arrayJoin([1, 2, 3]) AS id\n WHERE id = 1\n)\nWHERE id = 1 1 -SELECT \n id, \n subquery\nFROM \n(\n SELECT \n 1 AS id, \n CAST(1, \'UInt8\') AS subquery\n) +SELECT \n id,\n subquery\nFROM \n(\n SELECT \n 1 AS id,\n CAST(1, \'UInt8\') AS subquery\n) 1 1 -SELECT \n a, \n b\nFROM \n(\n SELECT \n toUInt64(sum(id) AS b) AS a, \n b\n FROM test_00597\n HAVING a = 3\n)\nWHERE a = 3 +SELECT \n a,\n b\nFROM \n(\n SELECT \n toUInt64(sum(id) AS b) AS a,\n b\n FROM test_00597\n HAVING a = 3\n)\nWHERE a = 3 3 3 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n name, \n value, \n min(id) AS id\n FROM test_00597\n GROUP BY \n date, \n name, \n value\n HAVING id = 1\n)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n name,\n value,\n min(id) AS id\n FROM test_00597\n GROUP BY \n date,\n name,\n value\n HAVING id = 1\n)\nWHERE id = 1 2000-01-01 1 test string 1 1 -SELECT \n a, \n b\nFROM \n(\n SELECT \n toUInt64(sum(id) AS b) AS a, \n b\n FROM test_00597 AS table_alias\n HAVING b = 3\n) AS outer_table_alias\nWHERE b = 3 +SELECT \n a,\n b\nFROM \n(\n SELECT \n toUInt64(sum(id) AS b) AS a,\n b\n FROM test_00597 AS table_alias\n HAVING b = 3\n) AS outer_table_alias\nWHERE b = 3 3 3 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n)\nWHERE id = 1 2000-01-01 1 test string 1 1 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM \n (\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n )\n WHERE id = 1\n)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM \n (\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n )\n WHERE id = 1\n)\nWHERE id = 1 2000-01-01 1 test string 1 1 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM \n (\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n ) AS b\n WHERE id = 1\n)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM \n (\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n ) AS b\n WHERE id = 1\n)\nWHERE id = 1 2000-01-01 1 test string 1 1 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n)\nWHERE id = 1 2000-01-01 1 test string 1 1 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM \n (\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n )\n WHERE id = 1\n)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM \n (\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n )\n WHERE id = 1\n)\nWHERE id = 1 2000-01-01 1 test string 1 1 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n) AS b\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n) AS b\nWHERE id = 1 2000-01-01 1 test string 1 1 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM \n (\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n ) AS a\n WHERE id = 1\n) AS b\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM \n (\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n ) AS a\n WHERE id = 1\n) AS b\nWHERE id = 1 2000-01-01 1 test string 1 1 -SELECT \n id, \n date, \n value\nFROM \n(\n SELECT \n id, \n date, \n min(value) AS value\n FROM test_00597\n WHERE id = 1\n GROUP BY \n id, \n date\n)\nWHERE id = 1 +SELECT \n id,\n date,\n value\nFROM \n(\n SELECT \n id,\n date,\n min(value) AS value\n FROM test_00597\n WHERE id = 1\n GROUP BY \n id,\n date\n)\nWHERE id = 1 1 2000-01-01 1 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n UNION ALL\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n UNION ALL\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n)\nWHERE id = 1 2000-01-01 1 test string 1 1 2000-01-01 1 test string 1 1 -SELECT \n date, \n id, \n name, \n value, \n date, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n)\nANY LEFT JOIN \n(\n SELECT id\n FROM test_00597\n) USING (id)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value,\n date,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n)\nANY LEFT JOIN \n(\n SELECT id\n FROM test_00597\n) USING (id)\nWHERE id = 1 2000-01-01 1 test string 1 1 2000-01-01 test string 1 1 -SELECT \n id, \n date, \n name, \n value\nFROM \n(\n SELECT toInt8(1) AS id\n)\nANY LEFT JOIN \n(\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n) AS test_00597 USING (id)\nWHERE value = 1 +SELECT \n id,\n date,\n name,\n value\nFROM \n(\n SELECT toInt8(1) AS id\n)\nANY LEFT JOIN \n(\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n) AS test_00597 USING (id)\nWHERE value = 1 1 2000-01-01 test string 1 1 SELECT value\nFROM \n(\n SELECT toInt8(1) AS id\n)\nANY LEFT JOIN test_00597 AS b USING (id)\nWHERE value = 1 1 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value, \n date, \n name, \n value\n FROM \n (\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n )\n ANY LEFT JOIN \n (\n SELECT id\n FROM test_00597\n ) USING (id)\n WHERE id = 1\n)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value,\n date,\n name,\n value\n FROM \n (\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n )\n ANY LEFT JOIN \n (\n SELECT id\n FROM test_00597\n ) USING (id)\n WHERE id = 1\n)\nWHERE id = 1 2000-01-01 1 test string 1 1 -SELECT \n date, \n id, \n name, \n value, \n b.date, \n b.name, \n b.value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n)\nANY LEFT JOIN \n(\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n) AS b USING (id)\nWHERE b.id = 1 +SELECT \n date,\n id,\n name,\n value,\n b.date,\n b.name,\n b.value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n)\nANY LEFT JOIN \n(\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n) AS b USING (id)\nWHERE b.id = 1 2000-01-01 1 test string 1 1 2000-01-01 test string 1 1 -SELECT \n id, \n date, \n name, \n value\nFROM \n(\n SELECT \n toInt8(1) AS id, \n toDate(\'2000-01-01\') AS date\n FROM system.numbers\n LIMIT 1\n)\nANY LEFT JOIN \n(\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n) AS b USING (date, id)\nWHERE b.date = toDate(\'2000-01-01\') +SELECT \n id,\n date,\n name,\n value\nFROM \n(\n SELECT \n toInt8(1) AS id,\n toDate(\'2000-01-01\') AS date\n FROM system.numbers\n LIMIT 1\n)\nANY LEFT JOIN \n(\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n) AS b USING (date, id)\nWHERE b.date = toDate(\'2000-01-01\') 1 2000-01-01 test string 1 1 -SELECT \n date, \n id, \n name, \n value, \n `b.date`, \n `b.id`, \n `b.name`, \n `b.value`\nFROM \n(\n SELECT \n date, \n id, \n name, \n value, \n b.date, \n b.id, \n b.name, \n b.value\n FROM \n (\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n ) AS a\n ANY LEFT JOIN \n (\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n ) AS b ON id = b.id\n WHERE id = 1\n)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value,\n `b.date`,\n `b.id`,\n `b.name`,\n `b.value`\nFROM \n(\n SELECT \n date,\n id,\n name,\n value,\n b.date,\n b.id,\n b.name,\n b.value\n FROM \n (\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n ) AS a\n ANY LEFT JOIN \n (\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n ) AS b ON id = b.id\n WHERE id = 1\n)\nWHERE id = 1 2000-01-01 1 test string 1 1 2000-01-01 1 test string 1 1 -SELECT \n date, \n id, \n name, \n value, \n r.date, \n r.name, \n r.value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n)\nSEMI LEFT JOIN \n(\n SELECT \n date, \n id, \n name, \n value\n FROM \n (\n SELECT \n date, \n id, \n name, \n value\n FROM test_00597\n WHERE id = 1\n )\n WHERE id = 1\n) AS r USING (id)\nWHERE r.id = 1 +SELECT \n date,\n id,\n name,\n value,\n r.date,\n r.name,\n r.value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n)\nSEMI LEFT JOIN \n(\n SELECT \n date,\n id,\n name,\n value\n FROM \n (\n SELECT \n date,\n id,\n name,\n value\n FROM test_00597\n WHERE id = 1\n )\n WHERE id = 1\n) AS r USING (id)\nWHERE r.id = 1 2000-01-01 1 test string 1 1 2000-01-01 test string 1 1 diff --git a/tests/queries/0_stateless/00751_default_databasename_for_view.reference b/tests/queries/0_stateless/00751_default_databasename_for_view.reference index 5ba1861e3ef..4814cc77b37 100644 --- a/tests/queries/0_stateless/00751_default_databasename_for_view.reference +++ b/tests/queries/0_stateless/00751_default_databasename_for_view.reference @@ -1,15 +1,15 @@ CREATE MATERIALIZED VIEW test_00751.t_mv_00751 ( - `date` Date, - `platform` Enum8('a' = 0, 'b' = 1), + `date` Date, + `platform` Enum8('a' = 0, 'b' = 1), `app` Enum8('a' = 0, 'b' = 1) ) ENGINE = MergeTree ORDER BY date SETTINGS index_granularity = 8192 AS SELECT - date, - platform, + date, + platform, app FROM test_00751.t_00751 WHERE (app = diff --git a/tests/queries/0_stateless/00826_cross_to_inner_join.reference b/tests/queries/0_stateless/00826_cross_to_inner_join.reference index 32b1c42ca2c..2a4b1487f20 100644 --- a/tests/queries/0_stateless/00826_cross_to_inner_join.reference +++ b/tests/queries/0_stateless/00826_cross_to_inner_join.reference @@ -35,18 +35,18 @@ comma nullable 1 1 1 1 2 2 1 2 cross -SELECT \n a, \n b, \n t2_00826.a, \n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON a = t2_00826.a\nWHERE a = t2_00826.a +SELECT \n a,\n b,\n t2_00826.a,\n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON a = t2_00826.a\nWHERE a = t2_00826.a cross nullable -SELECT \n a, \n b, \n t2_00826.a, \n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON a = t2_00826.a\nWHERE a = t2_00826.a +SELECT \n a,\n b,\n t2_00826.a,\n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON a = t2_00826.a\nWHERE a = t2_00826.a cross nullable vs not nullable -SELECT \n a, \n b, \n t2_00826.a, \n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON a = t2_00826.b\nWHERE a = t2_00826.b +SELECT \n a,\n b,\n t2_00826.a,\n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON a = t2_00826.b\nWHERE a = t2_00826.b cross self -SELECT \n a, \n b, \n y.a, \n y.b\nFROM t1_00826 AS x\nALL INNER JOIN t1_00826 AS y ON (a = y.a) AND (b = y.b)\nWHERE (a = y.a) AND (b = y.b) +SELECT \n a,\n b,\n y.a,\n y.b\nFROM t1_00826 AS x\nALL INNER JOIN t1_00826 AS y ON (a = y.a) AND (b = y.b)\nWHERE (a = y.a) AND (b = y.b) cross one table expr -SELECT \n a, \n b, \n t2_00826.a, \n t2_00826.b\nFROM t1_00826\nCROSS JOIN t2_00826\nWHERE a = b +SELECT \n a,\n b,\n t2_00826.a,\n t2_00826.b\nFROM t1_00826\nCROSS JOIN t2_00826\nWHERE a = b cross multiple ands -SELECT \n a, \n b, \n t2_00826.a, \n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON (a = t2_00826.a) AND (b = t2_00826.b)\nWHERE (a = t2_00826.a) AND (b = t2_00826.b) +SELECT \n a,\n b,\n t2_00826.a,\n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON (a = t2_00826.a) AND (b = t2_00826.b)\nWHERE (a = t2_00826.a) AND (b = t2_00826.b) cross and inside and -SELECT \n a, \n b, \n t2_00826.a, \n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON (a = t2_00826.a) AND (a = t2_00826.a) AND (a = t2_00826.a) AND (b = t2_00826.b)\nWHERE (a = t2_00826.a) AND ((a = t2_00826.a) AND ((a = t2_00826.a) AND (b = t2_00826.b))) +SELECT \n a,\n b,\n t2_00826.a,\n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON (a = t2_00826.a) AND (a = t2_00826.a) AND (a = t2_00826.a) AND (b = t2_00826.b)\nWHERE (a = t2_00826.a) AND ((a = t2_00826.a) AND ((a = t2_00826.a) AND (b = t2_00826.b))) cross split conjunction -SELECT \n a, \n b, \n t2_00826.a, \n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON (a = t2_00826.a) AND (b = t2_00826.b)\nWHERE (a = t2_00826.a) AND (b = t2_00826.b) AND (a >= 1) AND (t2_00826.b > 0) +SELECT \n a,\n b,\n t2_00826.a,\n t2_00826.b\nFROM t1_00826\nALL INNER JOIN t2_00826 ON (a = t2_00826.a) AND (b = t2_00826.b)\nWHERE (a = t2_00826.a) AND (b = t2_00826.b) AND (a >= 1) AND (t2_00826.b > 0) diff --git a/tests/queries/0_stateless/00849_multiple_comma_join.reference b/tests/queries/0_stateless/00849_multiple_comma_join.reference index 829a5d25e54..0f7d28b65a0 100644 --- a/tests/queries/0_stateless/00849_multiple_comma_join.reference +++ b/tests/queries/0_stateless/00849_multiple_comma_join.reference @@ -1,18 +1,18 @@ SELECT a\nFROM t1_00849\nCROSS JOIN t2_00849 SELECT a\nFROM t1_00849\nALL INNER JOIN t2_00849 ON a = t2_00849.a\nWHERE a = t2_00849.a SELECT a\nFROM t1_00849\nALL INNER JOIN t2_00849 ON b = t2_00849.b\nWHERE b = t2_00849.b -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n a AS `--t1_00849.a`, \n b, \n t2_00849.a AS `--t2_00849.a`, \n t2_00849.b\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.a` = `--t2_00849.a`\n) AS `--.s`\nALL INNER JOIN t3_00849 ON `--t1_00849.a` = a\nWHERE (`--t1_00849.a` = `--t2_00849.a`) AND (`--t1_00849.a` = a) -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n a AS `--t1_00849.a`, \n b AS `--t1_00849.b`, \n t2_00849.a, \n t2_00849.b AS `--t2_00849.b`\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.b` = `--t2_00849.b`\n) AS `--.s`\nALL INNER JOIN t3_00849 ON `--t1_00849.b` = b\nWHERE (`--t1_00849.b` = `--t2_00849.b`) AND (`--t1_00849.b` = b) -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`, \n b, \n `--t2_00849.a`, \n `t2_00849.b`, \n a AS `--t3_00849.a`, \n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`, \n b, \n t2_00849.a AS `--t2_00849.a`, \n t2_00849.b\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.a` = `--t2_00849.a`\n ) AS `--.s`\n ALL INNER JOIN t3_00849 ON `--t1_00849.a` = `--t3_00849.a`\n) AS `--.s`\nALL INNER JOIN t4_00849 ON `--t1_00849.a` = a\nWHERE (`--t1_00849.a` = `--t2_00849.a`) AND (`--t1_00849.a` = `--t3_00849.a`) AND (`--t1_00849.a` = a) -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`, \n `--t1_00849.b`, \n `t2_00849.a`, \n `--t2_00849.b`, \n a, \n b AS `--t3_00849.b`\n FROM \n (\n SELECT \n a AS `--t1_00849.a`, \n b AS `--t1_00849.b`, \n t2_00849.a, \n t2_00849.b AS `--t2_00849.b`\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.b` = `--t2_00849.b`\n ) AS `--.s`\n ALL INNER JOIN t3_00849 ON `--t1_00849.b` = `--t3_00849.b`\n) AS `--.s`\nALL INNER JOIN t4_00849 ON `--t1_00849.b` = b\nWHERE (`--t1_00849.b` = `--t2_00849.b`) AND (`--t1_00849.b` = `--t3_00849.b`) AND (`--t1_00849.b` = b) -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`, \n b, \n `--t2_00849.a`, \n `t2_00849.b`, \n a AS `--t3_00849.a`, \n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`, \n b, \n t2_00849.a AS `--t2_00849.a`, \n t2_00849.b\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t2_00849.a` = `--t1_00849.a`\n ) AS `--.s`\n ALL INNER JOIN t3_00849 ON `--t2_00849.a` = `--t3_00849.a`\n) AS `--.s`\nALL INNER JOIN t4_00849 ON `--t2_00849.a` = a\nWHERE (`--t2_00849.a` = `--t1_00849.a`) AND (`--t2_00849.a` = `--t3_00849.a`) AND (`--t2_00849.a` = a) -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`, \n b, \n `--t2_00849.a`, \n `t2_00849.b`, \n a AS `--t3_00849.a`, \n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`, \n b, \n t2_00849.a AS `--t2_00849.a`, \n t2_00849.b\n FROM t1_00849\n CROSS JOIN t2_00849\n ) AS `--.s`\n ALL INNER JOIN t3_00849 ON (`--t3_00849.a` = `--t1_00849.a`) AND (`--t3_00849.a` = `--t2_00849.a`)\n) AS `--.s`\nALL INNER JOIN t4_00849 ON `--t3_00849.a` = a\nWHERE (`--t3_00849.a` = `--t1_00849.a`) AND (`--t3_00849.a` = `--t2_00849.a`) AND (`--t3_00849.a` = a) -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`, \n b, \n `--t2_00849.a`, \n `t2_00849.b`, \n a AS `--t3_00849.a`, \n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`, \n b, \n t2_00849.a AS `--t2_00849.a`, \n t2_00849.b\n FROM t1_00849\n CROSS JOIN t2_00849\n ) AS `--.s`\n CROSS JOIN t3_00849\n) AS `--.s`\nALL INNER JOIN t4_00849 ON (a = `--t1_00849.a`) AND (a = `--t2_00849.a`) AND (a = `--t3_00849.a`)\nWHERE (a = `--t1_00849.a`) AND (a = `--t2_00849.a`) AND (a = `--t3_00849.a`) -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`, \n b, \n `--t2_00849.a`, \n `t2_00849.b`, \n a AS `--t3_00849.a`, \n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`, \n b, \n t2_00849.a AS `--t2_00849.a`, \n t2_00849.b\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.a` = `--t2_00849.a`\n ) AS `--.s`\n ALL INNER JOIN t3_00849 ON `--t2_00849.a` = `--t3_00849.a`\n) AS `--.s`\nALL INNER JOIN t4_00849 ON `--t3_00849.a` = a\nWHERE (`--t1_00849.a` = `--t2_00849.a`) AND (`--t2_00849.a` = `--t3_00849.a`) AND (`--t3_00849.a` = a) -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`, \n b, \n `t2_00849.a`, \n `t2_00849.b`, \n a, \n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`, \n b, \n t2_00849.a, \n t2_00849.b\n FROM t1_00849\n CROSS JOIN t2_00849\n ) AS `--.s`\n CROSS JOIN t3_00849\n) AS `--.s`\nCROSS JOIN t4_00849 -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`, \n b, \n `t2_00849.a`, \n `t2_00849.b`, \n a, \n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`, \n b, \n t2_00849.a, \n t2_00849.b\n FROM t1_00849\n CROSS JOIN t2_00849\n ) AS `--.s`\n CROSS JOIN t3_00849\n) AS `--.s`\nCROSS JOIN t4_00849 -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n a AS `--t1_00849.a`, \n b, \n t2_00849.a, \n t2_00849.b\n FROM t1_00849\n CROSS JOIN t2_00849\n) AS `--.s`\nCROSS JOIN t3_00849 -SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n a AS `--t1_00849.a`, \n b, \n t2_00849.a AS `--t2_00849.a`, \n t2_00849.b\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.a` = `--t2_00849.a`\n) AS `--.s`\nCROSS JOIN t3_00849 +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n a AS `--t1_00849.a`,\n b,\n t2_00849.a AS `--t2_00849.a`,\n t2_00849.b\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.a` = `--t2_00849.a`\n) AS `--.s`\nALL INNER JOIN t3_00849 ON `--t1_00849.a` = a\nWHERE (`--t1_00849.a` = `--t2_00849.a`) AND (`--t1_00849.a` = a) +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n a AS `--t1_00849.a`,\n b AS `--t1_00849.b`,\n t2_00849.a,\n t2_00849.b AS `--t2_00849.b`\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.b` = `--t2_00849.b`\n) AS `--.s`\nALL INNER JOIN t3_00849 ON `--t1_00849.b` = b\nWHERE (`--t1_00849.b` = `--t2_00849.b`) AND (`--t1_00849.b` = b) +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`,\n b,\n `--t2_00849.a`,\n `t2_00849.b`,\n a AS `--t3_00849.a`,\n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`,\n b,\n t2_00849.a AS `--t2_00849.a`,\n t2_00849.b\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.a` = `--t2_00849.a`\n ) AS `--.s`\n ALL INNER JOIN t3_00849 ON `--t1_00849.a` = `--t3_00849.a`\n) AS `--.s`\nALL INNER JOIN t4_00849 ON `--t1_00849.a` = a\nWHERE (`--t1_00849.a` = `--t2_00849.a`) AND (`--t1_00849.a` = `--t3_00849.a`) AND (`--t1_00849.a` = a) +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`,\n `--t1_00849.b`,\n `t2_00849.a`,\n `--t2_00849.b`,\n a,\n b AS `--t3_00849.b`\n FROM \n (\n SELECT \n a AS `--t1_00849.a`,\n b AS `--t1_00849.b`,\n t2_00849.a,\n t2_00849.b AS `--t2_00849.b`\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.b` = `--t2_00849.b`\n ) AS `--.s`\n ALL INNER JOIN t3_00849 ON `--t1_00849.b` = `--t3_00849.b`\n) AS `--.s`\nALL INNER JOIN t4_00849 ON `--t1_00849.b` = b\nWHERE (`--t1_00849.b` = `--t2_00849.b`) AND (`--t1_00849.b` = `--t3_00849.b`) AND (`--t1_00849.b` = b) +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`,\n b,\n `--t2_00849.a`,\n `t2_00849.b`,\n a AS `--t3_00849.a`,\n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`,\n b,\n t2_00849.a AS `--t2_00849.a`,\n t2_00849.b\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t2_00849.a` = `--t1_00849.a`\n ) AS `--.s`\n ALL INNER JOIN t3_00849 ON `--t2_00849.a` = `--t3_00849.a`\n) AS `--.s`\nALL INNER JOIN t4_00849 ON `--t2_00849.a` = a\nWHERE (`--t2_00849.a` = `--t1_00849.a`) AND (`--t2_00849.a` = `--t3_00849.a`) AND (`--t2_00849.a` = a) +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`,\n b,\n `--t2_00849.a`,\n `t2_00849.b`,\n a AS `--t3_00849.a`,\n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`,\n b,\n t2_00849.a AS `--t2_00849.a`,\n t2_00849.b\n FROM t1_00849\n CROSS JOIN t2_00849\n ) AS `--.s`\n ALL INNER JOIN t3_00849 ON (`--t3_00849.a` = `--t1_00849.a`) AND (`--t3_00849.a` = `--t2_00849.a`)\n) AS `--.s`\nALL INNER JOIN t4_00849 ON `--t3_00849.a` = a\nWHERE (`--t3_00849.a` = `--t1_00849.a`) AND (`--t3_00849.a` = `--t2_00849.a`) AND (`--t3_00849.a` = a) +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`,\n b,\n `--t2_00849.a`,\n `t2_00849.b`,\n a AS `--t3_00849.a`,\n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`,\n b,\n t2_00849.a AS `--t2_00849.a`,\n t2_00849.b\n FROM t1_00849\n CROSS JOIN t2_00849\n ) AS `--.s`\n CROSS JOIN t3_00849\n) AS `--.s`\nALL INNER JOIN t4_00849 ON (a = `--t1_00849.a`) AND (a = `--t2_00849.a`) AND (a = `--t3_00849.a`)\nWHERE (a = `--t1_00849.a`) AND (a = `--t2_00849.a`) AND (a = `--t3_00849.a`) +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`,\n b,\n `--t2_00849.a`,\n `t2_00849.b`,\n a AS `--t3_00849.a`,\n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`,\n b,\n t2_00849.a AS `--t2_00849.a`,\n t2_00849.b\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.a` = `--t2_00849.a`\n ) AS `--.s`\n ALL INNER JOIN t3_00849 ON `--t2_00849.a` = `--t3_00849.a`\n) AS `--.s`\nALL INNER JOIN t4_00849 ON `--t3_00849.a` = a\nWHERE (`--t1_00849.a` = `--t2_00849.a`) AND (`--t2_00849.a` = `--t3_00849.a`) AND (`--t3_00849.a` = a) +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`,\n b,\n `t2_00849.a`,\n `t2_00849.b`,\n a,\n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`,\n b,\n t2_00849.a,\n t2_00849.b\n FROM t1_00849\n CROSS JOIN t2_00849\n ) AS `--.s`\n CROSS JOIN t3_00849\n) AS `--.s`\nCROSS JOIN t4_00849 +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n `--t1_00849.a`,\n b,\n `t2_00849.a`,\n `t2_00849.b`,\n a,\n t3_00849.b\n FROM \n (\n SELECT \n a AS `--t1_00849.a`,\n b,\n t2_00849.a,\n t2_00849.b\n FROM t1_00849\n CROSS JOIN t2_00849\n ) AS `--.s`\n CROSS JOIN t3_00849\n) AS `--.s`\nCROSS JOIN t4_00849 +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n a AS `--t1_00849.a`,\n b,\n t2_00849.a,\n t2_00849.b\n FROM t1_00849\n CROSS JOIN t2_00849\n) AS `--.s`\nCROSS JOIN t3_00849 +SELECT `--t1_00849.a` AS `t1_00849.a`\nFROM \n(\n SELECT \n a AS `--t1_00849.a`,\n b,\n t2_00849.a AS `--t2_00849.a`,\n t2_00849.b\n FROM t1_00849\n ALL INNER JOIN t2_00849 ON `--t1_00849.a` = `--t2_00849.a`\n) AS `--.s`\nCROSS JOIN t3_00849 SELECT * FROM t1, t2 1 1 1 1 1 1 1 \N diff --git a/tests/queries/0_stateless/00849_multiple_comma_join_2.reference b/tests/queries/0_stateless/00849_multiple_comma_join_2.reference index 7875c1e9e86..f2e832123e0 100644 --- a/tests/queries/0_stateless/00849_multiple_comma_join_2.reference +++ b/tests/queries/0_stateless/00849_multiple_comma_join_2.reference @@ -1,18 +1,18 @@ SELECT a\nFROM t1\nCROSS JOIN t2 SELECT a\nFROM t1\nALL INNER JOIN t2 ON a = t2.a\nWHERE a = t2.a SELECT a\nFROM t1\nALL INNER JOIN t2 ON b = t2.b\nWHERE b = t2.b -SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n a AS `--t1.a`, \n t2.a AS `--t2.a`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n) AS `--.s`\nALL INNER JOIN t3 ON `--t1.a` = a\nWHERE (`--t1.a` = `--t2.a`) AND (`--t1.a` = a) -SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n b AS `--t1.b`, \n a AS `--t1.a`, \n t2.b AS `--t2.b`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.b` = `--t2.b`\n) AS `--.s`\nALL INNER JOIN t3 ON `--t1.b` = b\nWHERE (`--t1.b` = `--t2.b`) AND (`--t1.b` = b) -SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n `--t2.a`, \n a AS `--t3.a`\n FROM \n (\n SELECT \n a AS `--t1.a`, \n t2.a AS `--t2.a`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n ) AS `--.s`\n ALL INNER JOIN t3 ON `--t1.a` = `--t3.a`\n) AS `--.s`\nALL INNER JOIN t4 ON `--t1.a` = a\nWHERE (`--t1.a` = `--t2.a`) AND (`--t1.a` = `--t3.a`) AND (`--t1.a` = a) -SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.b`, \n `--t1.a`, \n `--t2.b`, \n b AS `--t3.b`\n FROM \n (\n SELECT \n b AS `--t1.b`, \n a AS `--t1.a`, \n t2.b AS `--t2.b`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.b` = `--t2.b`\n ) AS `--.s`\n ALL INNER JOIN t3 ON `--t1.b` = `--t3.b`\n) AS `--.s`\nALL INNER JOIN t4 ON `--t1.b` = b\nWHERE (`--t1.b` = `--t2.b`) AND (`--t1.b` = `--t3.b`) AND (`--t1.b` = b) -SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n `--t2.a`, \n a AS `--t3.a`\n FROM \n (\n SELECT \n a AS `--t1.a`, \n t2.a AS `--t2.a`\n FROM t1\n ALL INNER JOIN t2 ON `--t2.a` = `--t1.a`\n ) AS `--.s`\n ALL INNER JOIN t3 ON `--t2.a` = `--t3.a`\n) AS `--.s`\nALL INNER JOIN t4 ON `--t2.a` = a\nWHERE (`--t2.a` = `--t1.a`) AND (`--t2.a` = `--t3.a`) AND (`--t2.a` = a) -SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n `--t2.a`, \n a AS `--t3.a`\n FROM \n (\n SELECT \n a AS `--t1.a`, \n t2.a AS `--t2.a`\n FROM t1\n CROSS JOIN t2\n ) AS `--.s`\n ALL INNER JOIN t3 ON (`--t3.a` = `--t1.a`) AND (`--t3.a` = `--t2.a`)\n) AS `--.s`\nALL INNER JOIN t4 ON `--t3.a` = a\nWHERE (`--t3.a` = `--t1.a`) AND (`--t3.a` = `--t2.a`) AND (`--t3.a` = a) -SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n `--t2.a`, \n a AS `--t3.a`\n FROM \n (\n SELECT \n a AS `--t1.a`, \n t2.a AS `--t2.a`\n FROM t1\n CROSS JOIN t2\n ) AS `--.s`\n CROSS JOIN t3\n) AS `--.s`\nALL INNER JOIN t4 ON (a = `--t1.a`) AND (a = `--t2.a`) AND (a = `--t3.a`)\nWHERE (a = `--t1.a`) AND (a = `--t2.a`) AND (a = `--t3.a`) -SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`, \n `--t2.a`, \n a AS `--t3.a`\n FROM \n (\n SELECT \n a AS `--t1.a`, \n t2.a AS `--t2.a`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n ) AS `--.s`\n ALL INNER JOIN t3 ON `--t2.a` = `--t3.a`\n) AS `--.s`\nALL INNER JOIN t4 ON `--t3.a` = a\nWHERE (`--t1.a` = `--t2.a`) AND (`--t2.a` = `--t3.a`) AND (`--t3.a` = a) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n a AS `--t1.a`,\n t2.a AS `--t2.a`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n) AS `--.s`\nALL INNER JOIN t3 ON `--t1.a` = a\nWHERE (`--t1.a` = `--t2.a`) AND (`--t1.a` = a) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n b AS `--t1.b`,\n a AS `--t1.a`,\n t2.b AS `--t2.b`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.b` = `--t2.b`\n) AS `--.s`\nALL INNER JOIN t3 ON `--t1.b` = b\nWHERE (`--t1.b` = `--t2.b`) AND (`--t1.b` = b) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`,\n `--t2.a`,\n a AS `--t3.a`\n FROM \n (\n SELECT \n a AS `--t1.a`,\n t2.a AS `--t2.a`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n ) AS `--.s`\n ALL INNER JOIN t3 ON `--t1.a` = `--t3.a`\n) AS `--.s`\nALL INNER JOIN t4 ON `--t1.a` = a\nWHERE (`--t1.a` = `--t2.a`) AND (`--t1.a` = `--t3.a`) AND (`--t1.a` = a) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.b`,\n `--t1.a`,\n `--t2.b`,\n b AS `--t3.b`\n FROM \n (\n SELECT \n b AS `--t1.b`,\n a AS `--t1.a`,\n t2.b AS `--t2.b`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.b` = `--t2.b`\n ) AS `--.s`\n ALL INNER JOIN t3 ON `--t1.b` = `--t3.b`\n) AS `--.s`\nALL INNER JOIN t4 ON `--t1.b` = b\nWHERE (`--t1.b` = `--t2.b`) AND (`--t1.b` = `--t3.b`) AND (`--t1.b` = b) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`,\n `--t2.a`,\n a AS `--t3.a`\n FROM \n (\n SELECT \n a AS `--t1.a`,\n t2.a AS `--t2.a`\n FROM t1\n ALL INNER JOIN t2 ON `--t2.a` = `--t1.a`\n ) AS `--.s`\n ALL INNER JOIN t3 ON `--t2.a` = `--t3.a`\n) AS `--.s`\nALL INNER JOIN t4 ON `--t2.a` = a\nWHERE (`--t2.a` = `--t1.a`) AND (`--t2.a` = `--t3.a`) AND (`--t2.a` = a) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`,\n `--t2.a`,\n a AS `--t3.a`\n FROM \n (\n SELECT \n a AS `--t1.a`,\n t2.a AS `--t2.a`\n FROM t1\n CROSS JOIN t2\n ) AS `--.s`\n ALL INNER JOIN t3 ON (`--t3.a` = `--t1.a`) AND (`--t3.a` = `--t2.a`)\n) AS `--.s`\nALL INNER JOIN t4 ON `--t3.a` = a\nWHERE (`--t3.a` = `--t1.a`) AND (`--t3.a` = `--t2.a`) AND (`--t3.a` = a) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`,\n `--t2.a`,\n a AS `--t3.a`\n FROM \n (\n SELECT \n a AS `--t1.a`,\n t2.a AS `--t2.a`\n FROM t1\n CROSS JOIN t2\n ) AS `--.s`\n CROSS JOIN t3\n) AS `--.s`\nALL INNER JOIN t4 ON (a = `--t1.a`) AND (a = `--t2.a`) AND (a = `--t3.a`)\nWHERE (a = `--t1.a`) AND (a = `--t2.a`) AND (a = `--t3.a`) +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n `--t1.a`,\n `--t2.a`,\n a AS `--t3.a`\n FROM \n (\n SELECT \n a AS `--t1.a`,\n t2.a AS `--t2.a`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n ) AS `--.s`\n ALL INNER JOIN t3 ON `--t2.a` = `--t3.a`\n) AS `--.s`\nALL INNER JOIN t4 ON `--t3.a` = a\nWHERE (`--t1.a` = `--t2.a`) AND (`--t2.a` = `--t3.a`) AND (`--t3.a` = a) SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT `--t1.a`\n FROM \n (\n SELECT a AS `--t1.a`\n FROM t1\n CROSS JOIN t2\n ) AS `--.s`\n CROSS JOIN t3\n) AS `--.s`\nCROSS JOIN t4 SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT `--t1.a`\n FROM \n (\n SELECT a AS `--t1.a`\n FROM t1\n CROSS JOIN t2\n ) AS `--.s`\n CROSS JOIN t3\n) AS `--.s`\nCROSS JOIN t4 SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT a AS `--t1.a`\n FROM t1\n CROSS JOIN t2\n) AS `--.s`\nCROSS JOIN t3 -SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n a AS `--t1.a`, \n t2.a AS `--t2.a`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n) AS `--.s`\nCROSS JOIN t3 +SELECT `--t1.a` AS `t1.a`\nFROM \n(\n SELECT \n a AS `--t1.a`,\n t2.a AS `--t2.a`\n FROM t1\n ALL INNER JOIN t2 ON `--t1.a` = `--t2.a`\n) AS `--.s`\nCROSS JOIN t3 SELECT * FROM t1, t2 1 1 1 1 1 1 1 \N diff --git a/tests/queries/0_stateless/00908_analyze_query.reference b/tests/queries/0_stateless/00908_analyze_query.reference index a8619cfcd4b..66db6f5a2e4 100644 --- a/tests/queries/0_stateless/00908_analyze_query.reference +++ b/tests/queries/0_stateless/00908_analyze_query.reference @@ -1 +1 @@ -SELECT \n a, \n b\nFROM a +SELECT \n a,\n b\nFROM a diff --git a/tests/queries/0_stateless/00957_format_with_clashed_aliases.reference b/tests/queries/0_stateless/00957_format_with_clashed_aliases.reference index d3f7a9aa18b..d1c8033b363 100644 --- a/tests/queries/0_stateless/00957_format_with_clashed_aliases.reference +++ b/tests/queries/0_stateless/00957_format_with_clashed_aliases.reference @@ -1,5 +1,5 @@ SELECT - 1 AS x, + 1 AS x, x.y FROM ( diff --git a/tests/queries/0_stateless/00958_format_of_tuple_array_element.reference b/tests/queries/0_stateless/00958_format_of_tuple_array_element.reference index 7265311960f..eaea02ba40b 100644 --- a/tests/queries/0_stateless/00958_format_of_tuple_array_element.reference +++ b/tests/queries/0_stateless/00958_format_of_tuple_array_element.reference @@ -1,9 +1,9 @@ SELECT - (x.1)[1], - (((x[1]).1)[1]).1, - (NOT x)[1], - -(x[1]), - (-x)[1], - (NOT x).1, - -(x.1), + (x.1)[1], + (((x[1]).1)[1]).1, + (NOT x)[1], + -(x[1]), + (-x)[1], + (NOT x).1, + -(x.1), (-x).1 diff --git a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference index bd132202979..c797226d832 100644 --- a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference +++ b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference @@ -1,10 +1,10 @@ -SELECT \n k, \n v, \n d, \n i\nFROM \n(\n SELECT \n t.1 AS k, \n t.2 AS v, \n runningDifference(v) AS d, \n runningDifference(cityHash64(t.1)) AS i\n FROM \n (\n SELECT arrayJoin([(\'a\', 1), (\'a\', 2), (\'a\', 3), (\'b\', 11), (\'b\', 13), (\'b\', 15)]) AS t\n )\n)\nWHERE i = 0 +SELECT \n k,\n v,\n d,\n i\nFROM \n(\n SELECT \n t.1 AS k,\n t.2 AS v,\n runningDifference(v) AS d,\n runningDifference(cityHash64(t.1)) AS i\n FROM \n (\n SELECT arrayJoin([(\'a\', 1), (\'a\', 2), (\'a\', 3), (\'b\', 11), (\'b\', 13), (\'b\', 15)]) AS t\n )\n)\nWHERE i = 0 a 1 0 0 a 2 1 0 a 3 1 0 b 13 2 0 b 15 2 0 -SELECT \n co, \n co2, \n co3, \n num\nFROM \n(\n SELECT \n co, \n co2, \n co3, \n count() AS num\n FROM \n (\n SELECT \n 1 AS co, \n 2 AS co2, \n 3 AS co3\n )\n GROUP BY \n co, \n co2, \n co3\n WITH CUBE\n HAVING (co2 != 2) AND (co != 0)\n)\nWHERE (co != 0) AND (co2 != 2) +SELECT \n co,\n co2,\n co3,\n num\nFROM \n(\n SELECT \n co,\n co2,\n co3,\n count() AS num\n FROM \n (\n SELECT \n 1 AS co,\n 2 AS co2,\n 3 AS co3\n )\n GROUP BY \n co,\n co2,\n co3\n WITH CUBE\n HAVING (co2 != 2) AND (co != 0)\n)\nWHERE (co != 0) AND (co2 != 2) 1 0 3 1 1 0 0 1 SELECT alias AS name\nFROM \n(\n SELECT name AS alias\n FROM system.settings\n WHERE alias = \'enable_optimize_predicate_expression\'\n)\nANY INNER JOIN \n(\n SELECT name\n FROM system.settings\n) USING (name)\nWHERE name = \'enable_optimize_predicate_expression\' @@ -12,8 +12,8 @@ enable_optimize_predicate_expression 1 val11 val21 val31 SELECT ccc\nFROM \n(\n SELECT 1 AS ccc\n WHERE 0\n UNION ALL\n SELECT ccc\n FROM \n (\n SELECT 2 AS ccc\n )\n ANY INNER JOIN \n (\n SELECT 2 AS ccc\n ) USING (ccc)\n WHERE ccc > 1\n)\nWHERE ccc > 1 2 -SELECT \n ts, \n id, \n id_b, \n b.ts, \n b.id, \n id_c\nFROM \n(\n SELECT \n ts, \n id, \n id_b\n FROM A\n WHERE ts <= toDateTime(\'1970-01-01 03:00:00\')\n) AS a\nALL LEFT JOIN B AS b ON b.id = id_b\nWHERE ts <= toDateTime(\'1970-01-01 03:00:00\') -SELECT \n ts AS `--a.ts`, \n id AS `--a.id`, \n id_b AS `--a.id_b`, \n b.ts AS `--b.ts`, \n b.id AS `--b.id`, \n id_c AS `--b.id_c`\nFROM \n(\n SELECT \n ts, \n id, \n id_b\n FROM A\n WHERE ts <= toDateTime(\'1970-01-01 03:00:00\')\n) AS a\nALL LEFT JOIN B AS b ON `--b.id` = `--a.id_b`\nWHERE `--a.ts` <= toDateTime(\'1970-01-01 03:00:00\') +SELECT \n ts,\n id,\n id_b,\n b.ts,\n b.id,\n id_c\nFROM \n(\n SELECT \n ts,\n id,\n id_b\n FROM A\n WHERE ts <= toDateTime(\'1970-01-01 03:00:00\')\n) AS a\nALL LEFT JOIN B AS b ON b.id = id_b\nWHERE ts <= toDateTime(\'1970-01-01 03:00:00\') +SELECT \n ts AS `--a.ts`,\n id AS `--a.id`,\n id_b AS `--a.id_b`,\n b.ts AS `--b.ts`,\n b.id AS `--b.id`,\n id_c AS `--b.id_c`\nFROM \n(\n SELECT \n ts,\n id,\n id_b\n FROM A\n WHERE ts <= toDateTime(\'1970-01-01 03:00:00\')\n) AS a\nALL LEFT JOIN B AS b ON `--b.id` = `--a.id_b`\nWHERE `--a.ts` <= toDateTime(\'1970-01-01 03:00:00\') 2 3 3 4 4 5 @@ -24,6 +24,6 @@ SELECT \n ts AS `--a.ts`, \n id AS `--a.id`, \n id_b AS `--a.id_b`, \n 4 5 SELECT dummy\nFROM \n(\n SELECT dummy\n FROM system.one\n WHERE arrayMap(x -> (x + 1), [dummy]) = [1]\n)\nWHERE arrayMap(x -> (x + 1), [dummy]) = [1] 0 -SELECT \n id, \n value, \n value_1\nFROM \n(\n SELECT \n 1 AS id, \n 2 AS value\n)\nALL INNER JOIN \n(\n SELECT \n 1 AS id, \n 3 AS value_1\n) USING (id)\nWHERE arrayMap(x -> ((x + value) + value_1), [1]) = [6] +SELECT \n id,\n value,\n value_1\nFROM \n(\n SELECT \n 1 AS id,\n 2 AS value\n)\nALL INNER JOIN \n(\n SELECT \n 1 AS id,\n 3 AS value_1\n) USING (id)\nWHERE arrayMap(x -> ((x + value) + value_1), [1]) = [6] 1 2 3 SELECT dummy\nFROM system.one\nWHERE (dummy > 0) AND (dummy < 0) diff --git a/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference b/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference index 1e92e7b8596..e2c3b5dab4a 100644 --- a/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference +++ b/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference @@ -1,4 +1,4 @@ -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 1\n)\nWHERE id = 1 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 2\n)\nWHERE id = 2 -SELECT id\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 1\n)\nWHERE id = 1 -SELECT id\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 1\n) AS s\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM default.test\n WHERE id = 1\n)\nWHERE id = 1 +SELECT \n date,\n id,\n name,\n value\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM default.test\n WHERE id = 2\n)\nWHERE id = 2 +SELECT id\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM default.test\n WHERE id = 1\n)\nWHERE id = 1 +SELECT id\nFROM \n(\n SELECT \n date,\n id,\n name,\n value\n FROM default.test\n WHERE id = 1\n) AS s\nWHERE id = 1 diff --git a/tests/queries/0_stateless/01083_cross_to_inner_with_like.reference b/tests/queries/0_stateless/01083_cross_to_inner_with_like.reference index 92b51afb544..e6ebffcae9c 100644 --- a/tests/queries/0_stateless/01083_cross_to_inner_with_like.reference +++ b/tests/queries/0_stateless/01083_cross_to_inner_with_like.reference @@ -1,3 +1,3 @@ -SELECT \n k, \n r.k, \n name\nFROM n\nALL INNER JOIN r ON k = r.k\nWHERE (k = r.k) AND (name = \'A\') -SELECT \n k, \n r.k, \n name\nFROM n\nALL INNER JOIN r ON k = r.k\nWHERE (k = r.k) AND (name LIKE \'A%\') -SELECT \n k, \n r.k, \n name\nFROM n\nALL INNER JOIN r ON k = r.k\nWHERE (k = r.k) AND (name NOT LIKE \'A%\') +SELECT \n k,\n r.k,\n name\nFROM n\nALL INNER JOIN r ON k = r.k\nWHERE (k = r.k) AND (name = \'A\') +SELECT \n k,\n r.k,\n name\nFROM n\nALL INNER JOIN r ON k = r.k\nWHERE (k = r.k) AND (name LIKE \'A%\') +SELECT \n k,\n r.k,\n name\nFROM n\nALL INNER JOIN r ON k = r.k\nWHERE (k = r.k) AND (name NOT LIKE \'A%\') diff --git a/tests/queries/0_stateless/01278_format_multiple_queries.reference b/tests/queries/0_stateless/01278_format_multiple_queries.reference index cba2cc7b320..b12e3b30f0c 100644 --- a/tests/queries/0_stateless/01278_format_multiple_queries.reference +++ b/tests/queries/0_stateless/01278_format_multiple_queries.reference @@ -1,5 +1,5 @@ SELECT - a, + a, b AS x FROM table AS t INNER JOIN table2 AS t2 ON t.id = t2.t_id @@ -7,8 +7,8 @@ WHERE 1 = 1 ; SELECT - a, - b AS x, + a, + b AS x, if(x = 0, a, b) FROM table2 AS t WHERE t.id != 0 From 9f8c156fd23132d419618bbd8668f83245dc49fb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 2 Jun 2020 02:35:44 +0300 Subject: [PATCH 020/329] Remove debug output --- src/Parsers/ASTNameTypePair.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Parsers/ASTNameTypePair.cpp b/src/Parsers/ASTNameTypePair.cpp index 6c41d35315c..35493eb77d1 100644 --- a/src/Parsers/ASTNameTypePair.cpp +++ b/src/Parsers/ASTNameTypePair.cpp @@ -24,10 +24,8 @@ void ASTNameTypePair::formatImpl(const FormatSettings & settings, FormatState & { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); - settings.ostr << '#'; settings.ostr << indent_str << backQuoteIfNeed(name) << ' '; type->formatImpl(settings, state, frame); - settings.ostr << '#'; } } From 36c23e240df0c70917abfe38f1d9b910e8ed64c1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 2 Jun 2020 02:41:41 +0300 Subject: [PATCH 021/329] Update some tests --- .../00061_merge_tree_alter.reference | 20 ++++++------ .../queries/0_stateless/00642_cast.reference | 2 +- .../00643_cast_zookeeper.reference | 2 +- .../00725_comment_columns.reference | 12 +++---- .../00725_ipv4_ipv6_domains.reference | 4 +-- .../00753_comment_columns_zookeeper.reference | 4 +-- ...4_alter_modify_column_partitions.reference | 4 +-- .../00754_alter_modify_order_by.reference | 2 +- ...fy_order_by_replicated_zookeeper.reference | 4 +-- ...4_test_custom_compression_codecs.reference | 4 +-- ...m_compression_codes_log_storages.reference | 8 ++--- .../0_stateless/00836_indices_alter.reference | 10 +++--- ...dices_alter_replicated_zookeeper.reference | 24 +++++++------- .../0_stateless/00933_alter_ttl.reference | 2 +- .../0_stateless/00933_ttl_simple.reference | 8 ++--- .../00980_merge_alter_settings.reference | 10 +++--- ...keeper_merge_tree_alter_settings.reference | 12 +++---- .../00998_constraints_all_tables.reference | 4 +-- ...age_odbc_parsing_exception_check.reference | 2 +- .../01055_compact_parts_1.reference | 4 +-- .../01069_database_memory.reference | 2 +- .../01070_alter_with_ttl.reference | 4 +-- .../01079_alter_default_zookeeper.reference | 16 +++++----- .../01079_bad_alters_zookeeper.reference | 4 +-- ..._expressions_in_engine_arguments.reference | 14 ++++---- ...1135_default_and_alter_zookeeper.reference | 2 +- ...13_alter_rename_column_zookeeper.reference | 4 +-- .../01213_alter_rename_nested.reference | 6 ++-- ...er_rename_with_default_zookeeper.reference | 10 +++--- .../01213_alter_table_rename_nested.reference | 4 +-- ...01224_no_superfluous_dict_reload.reference | 2 +- ...how_create_table_from_dictionary.reference | 2 +- ...9_bad_arguments_for_bloom_filter.reference | 6 ++-- .../01272_suspicious_codecs.reference | 32 +++++++++---------- ...alter_rename_column_default_expr.reference | 4 +-- ..._rename_column_materialized_expr.reference | 4 +-- ...7_alter_rename_column_constraint.reference | 4 +-- ...name_column_constraint_zookeeper.reference | 4 +-- .../01278_alter_rename_combination.reference | 8 ++--- ...1_alter_rename_and_other_renames.reference | 12 +++---- 40 files changed, 143 insertions(+), 143 deletions(-) diff --git a/tests/queries/0_stateless/00061_merge_tree_alter.reference b/tests/queries/0_stateless/00061_merge_tree_alter.reference index b609bc257f1..dcc44b9bd81 100644 --- a/tests/queries/0_stateless/00061_merge_tree_alter.reference +++ b/tests/queries/0_stateless/00061_merge_tree_alter.reference @@ -1,14 +1,14 @@ d Date k UInt64 i32 Int32 -CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32\n)\nENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 10 42 d Date k UInt64 i32 Int32 n.ui8 Array(UInt8) n.s Array(String) -CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.ui8` Array(UInt8), \n `n.s` Array(String)\n)\nENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `n.ui8` Array(UInt8),\n `n.s` Array(String)\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 8 40 [1,2,3] ['12','13','14'] 2015-01-01 10 42 [] [] d Date @@ -17,7 +17,7 @@ i32 Int32 n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) -CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 7 39 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 2015-01-01 8 40 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 10 42 [] [] [] @@ -28,7 +28,7 @@ n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) s String DEFAULT \'0\' -CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date), \n `s` String DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date),\n `s` String DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 [10,20,30] ['asd','qwe','qwe'] ['2000-01-01','2000-01-01','2000-01-03'] 100500 2015-01-01 7 39 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 0 2015-01-01 8 40 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 0 @@ -39,7 +39,7 @@ i32 Int32 n.ui8 Array(UInt8) n.s Array(String) s Int64 DEFAULT \'0\' -CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` Int64 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` Int64 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 [10,20,30] ['asd','qwe','qwe'] 100500 2015-01-01 7 39 [10,20,30] ['120','130','140'] 0 2015-01-01 8 40 [1,2,3] ['12','13','14'] 0 @@ -51,7 +51,7 @@ n.ui8 Array(UInt8) n.s Array(String) s UInt32 DEFAULT \'0\' n.d Array(Date) -CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\', \n `n.d` Array(Date)\n)\nENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\',\n `n.d` Array(Date)\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 [10,20,30] ['asd','qwe','qwe'] 100500 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 7 39 [10,20,30] ['120','130','140'] 0 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 8 40 [1,2,3] ['12','13','14'] 0 ['0000-00-00','0000-00-00','0000-00-00'] @@ -65,7 +65,7 @@ k UInt64 i32 Int32 n.s Array(String) s UInt32 DEFAULT \'0\' -CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 ['asd','qwe','qwe'] 100500 2015-01-01 7 39 ['120','130','140'] 0 2015-01-01 8 40 ['12','13','14'] 0 @@ -74,7 +74,7 @@ d Date k UInt64 i32 Int32 s UInt32 DEFAULT \'0\' -CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 100500 2015-01-01 7 39 0 2015-01-01 8 40 0 @@ -85,7 +85,7 @@ i32 Int32 s UInt32 DEFAULT \'0\' n.s Array(String) n.d Array(Date) -CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `s` UInt32 DEFAULT \'0\', \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `s` UInt32 DEFAULT \'0\',\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 100500 [] [] 2015-01-01 7 39 0 [] [] 2015-01-01 8 40 0 [] [] @@ -94,7 +94,7 @@ d Date k UInt64 i32 Int32 s UInt32 DEFAULT \'0\' -CREATE TABLE default.alter_00061\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) +CREATE TABLE default.alter_00061\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 100500 2015-01-01 7 39 0 2015-01-01 8 40 0 diff --git a/tests/queries/0_stateless/00642_cast.reference b/tests/queries/0_stateless/00642_cast.reference index 907861c1784..7f5333f590e 100644 --- a/tests/queries/0_stateless/00642_cast.reference +++ b/tests/queries/0_stateless/00642_cast.reference @@ -9,7 +9,7 @@ hello 1970-01-01 00:00:01 CREATE TABLE default.cast ( - `x` UInt8, + `x` UInt8, `e` Enum8('hello' = 1, 'world' = 2) DEFAULT CAST(x, 'Enum8(\'hello\' = 1, \'world\' = 2)') ) ENGINE = MergeTree diff --git a/tests/queries/0_stateless/00643_cast_zookeeper.reference b/tests/queries/0_stateless/00643_cast_zookeeper.reference index b79eb07aee3..226390d8510 100644 --- a/tests/queries/0_stateless/00643_cast_zookeeper.reference +++ b/tests/queries/0_stateless/00643_cast_zookeeper.reference @@ -1,6 +1,6 @@ CREATE TABLE test.cast1 ( - `x` UInt8, + `x` UInt8, `e` Enum8('hello' = 1, 'world' = 2) DEFAULT CAST(x, 'Enum8(\'hello\' = 1, \'world\' = 2)') ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_cast', 'r1') diff --git a/tests/queries/0_stateless/00725_comment_columns.reference b/tests/queries/0_stateless/00725_comment_columns.reference index 86794581daf..7543f5854d7 100644 --- a/tests/queries/0_stateless/00725_comment_columns.reference +++ b/tests/queries/0_stateless/00725_comment_columns.reference @@ -1,4 +1,4 @@ -CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 DEFAULT 1 COMMENT \'comment 1\', \n `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2\', \n `third_column` UInt8 ALIAS second_column COMMENT \'comment 3\', \n `fourth_column` UInt8 COMMENT \'comment 4\', \n `fifth_column` UInt8\n)\nENGINE = TinyLog +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 DEFAULT 1 COMMENT \'comment 1\',\n `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2\',\n `third_column` UInt8 ALIAS second_column COMMENT \'comment 3\',\n `fourth_column` UInt8 COMMENT \'comment 4\',\n `fifth_column` UInt8\n)\nENGINE = TinyLog first_column UInt8 DEFAULT 1 comment 1 second_column UInt8 MATERIALIZED first_column comment 2 third_column UInt8 ALIAS second_column comment 3 @@ -11,7 +11,7 @@ fifth_column UInt8 │ check_query_comment_column │ fourth_column │ comment 4 │ │ check_query_comment_column │ fifth_column │ │ └────────────────────────────┴───────────────┴───────────┘ -CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 DEFAULT 1 COMMENT \'comment 1_1\', \n `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2_1\', \n `third_column` UInt8 ALIAS second_column COMMENT \'comment 3_1\', \n `fourth_column` UInt8 COMMENT \'comment 4_1\', \n `fifth_column` UInt8 COMMENT \'comment 5_1\'\n)\nENGINE = TinyLog +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 DEFAULT 1 COMMENT \'comment 1_1\',\n `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2_1\',\n `third_column` UInt8 ALIAS second_column COMMENT \'comment 3_1\',\n `fourth_column` UInt8 COMMENT \'comment 4_1\',\n `fifth_column` UInt8 COMMENT \'comment 5_1\'\n)\nENGINE = TinyLog ┌─table──────────────────────┬─name──────────┬─comment─────┐ │ check_query_comment_column │ first_column │ comment 1_2 │ │ check_query_comment_column │ second_column │ comment 2_2 │ @@ -19,8 +19,8 @@ CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 DEF │ check_query_comment_column │ fourth_column │ comment 4_2 │ │ check_query_comment_column │ fifth_column │ comment 5_2 │ └────────────────────────────┴───────────────┴─────────────┘ -CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 DEFAULT 1 COMMENT \'comment 1_2\', \n `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2_2\', \n `third_column` UInt8 ALIAS second_column COMMENT \'comment 3_2\', \n `fourth_column` UInt8 COMMENT \'comment 4_2\', \n `fifth_column` UInt8 COMMENT \'comment 5_2\'\n)\nENGINE = TinyLog -CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 COMMENT \'comment 1\', \n `second_column` UInt8 COMMENT \'comment 2\', \n `third_column` UInt8 COMMENT \'comment 3\'\n)\nENGINE = MergeTree()\nPARTITION BY second_column\nORDER BY first_column\nSAMPLE BY first_column\nSETTINGS index_granularity = 8192 +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 DEFAULT 1 COMMENT \'comment 1_2\',\n `second_column` UInt8 MATERIALIZED first_column COMMENT \'comment 2_2\',\n `third_column` UInt8 ALIAS second_column COMMENT \'comment 3_2\',\n `fourth_column` UInt8 COMMENT \'comment 4_2\',\n `fifth_column` UInt8 COMMENT \'comment 5_2\'\n)\nENGINE = TinyLog +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 COMMENT \'comment 1\',\n `second_column` UInt8 COMMENT \'comment 2\',\n `third_column` UInt8 COMMENT \'comment 3\'\n)\nENGINE = MergeTree()\nPARTITION BY second_column\nORDER BY first_column\nSAMPLE BY first_column\nSETTINGS index_granularity = 8192 first_column UInt8 comment 1 second_column UInt8 comment 2 third_column UInt8 comment 3 @@ -29,8 +29,8 @@ third_column UInt8 comment 3 │ check_query_comment_column │ second_column │ comment 2 │ │ check_query_comment_column │ third_column │ comment 3 │ └────────────────────────────┴───────────────┴───────────┘ -CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 COMMENT \'comment 1_2\', \n `second_column` UInt8 COMMENT \'comment 2_2\', \n `third_column` UInt8 COMMENT \'comment 3_2\'\n)\nENGINE = MergeTree()\nPARTITION BY second_column\nORDER BY first_column\nSAMPLE BY first_column\nSETTINGS index_granularity = 8192 -CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 COMMENT \'comment 1_3\', \n `second_column` UInt8 COMMENT \'comment 2_3\', \n `third_column` UInt8 COMMENT \'comment 3_3\'\n)\nENGINE = MergeTree()\nPARTITION BY second_column\nORDER BY first_column\nSAMPLE BY first_column\nSETTINGS index_granularity = 8192 +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 COMMENT \'comment 1_2\',\n `second_column` UInt8 COMMENT \'comment 2_2\',\n `third_column` UInt8 COMMENT \'comment 3_2\'\n)\nENGINE = MergeTree()\nPARTITION BY second_column\nORDER BY first_column\nSAMPLE BY first_column\nSETTINGS index_granularity = 8192 +CREATE TABLE default.check_query_comment_column\n(\n `first_column` UInt8 COMMENT \'comment 1_3\',\n `second_column` UInt8 COMMENT \'comment 2_3\',\n `third_column` UInt8 COMMENT \'comment 3_3\'\n)\nENGINE = MergeTree()\nPARTITION BY second_column\nORDER BY first_column\nSAMPLE BY first_column\nSETTINGS index_granularity = 8192 ┌─table──────────────────────┬─name──────────┬─comment─────┐ │ check_query_comment_column │ first_column │ comment 1_3 │ │ check_query_comment_column │ second_column │ comment 2_3 │ diff --git a/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference b/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference index 28051d15f65..69804e6cd24 100644 --- a/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference +++ b/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference @@ -1,4 +1,4 @@ -CREATE TABLE default.ipv4_test\n(\n `ipv4_` IPv4\n)\nENGINE = Memory +CREATE TABLE default.ipv4_test\n(`ipv4_` IPv4\n)\nENGINE = Memory 0.0.0.0 00 8.8.8.8 08080808 127.0.0.1 7F000001 @@ -10,7 +10,7 @@ CREATE TABLE default.ipv4_test\n(\n `ipv4_` IPv4\n)\nENGINE = Memory > 127.0.0.1 255.255.255.255 = 127.0.0.1 127.0.0.1 euqality of IPv4-mapped IPv6 value and IPv4 promoted to IPv6 with function: 1 -CREATE TABLE default.ipv6_test\n(\n `ipv6_` IPv6\n)\nENGINE = Memory +CREATE TABLE default.ipv6_test\n(`ipv6_` IPv6\n)\nENGINE = Memory :: 00000000000000000000000000000000 :: 00000000000000000000000000000000 ::ffff:8.8.8.8 00000000000000000000FFFF08080808 diff --git a/tests/queries/0_stateless/00753_comment_columns_zookeeper.reference b/tests/queries/0_stateless/00753_comment_columns_zookeeper.reference index d2705135440..5d8c5dc9f72 100644 --- a/tests/queries/0_stateless/00753_comment_columns_zookeeper.reference +++ b/tests/queries/0_stateless/00753_comment_columns_zookeeper.reference @@ -1,6 +1,6 @@ -CREATE TABLE default.check_comments\n(\n `column_name1` UInt8 DEFAULT 1 COMMENT \'comment\', \n `column_name2` UInt8 COMMENT \'non default comment\'\n)\nENGINE = ReplicatedMergeTree(\'clickhouse/tables/test_comments\', \'r1\')\nORDER BY column_name1\nSETTINGS index_granularity = 8192 +CREATE TABLE default.check_comments\n(\n `column_name1` UInt8 DEFAULT 1 COMMENT \'comment\',\n `column_name2` UInt8 COMMENT \'non default comment\'\n)\nENGINE = ReplicatedMergeTree(\'clickhouse/tables/test_comments\', \'r1\')\nORDER BY column_name1\nSETTINGS index_granularity = 8192 column_name1 UInt8 DEFAULT 1 comment column_name2 UInt8 non default comment -CREATE TABLE default.check_comments\n(\n `column_name1` UInt8 DEFAULT 1 COMMENT \'another comment\', \n `column_name2` UInt8 COMMENT \'non default comment\'\n)\nENGINE = ReplicatedMergeTree(\'clickhouse/tables/test_comments\', \'r1\')\nORDER BY column_name1\nSETTINGS index_granularity = 8192 +CREATE TABLE default.check_comments\n(\n `column_name1` UInt8 DEFAULT 1 COMMENT \'another comment\',\n `column_name2` UInt8 COMMENT \'non default comment\'\n)\nENGINE = ReplicatedMergeTree(\'clickhouse/tables/test_comments\', \'r1\')\nORDER BY column_name1\nSETTINGS index_granularity = 8192 column_name1 UInt8 DEFAULT 1 another comment column_name2 UInt8 non default comment diff --git a/tests/queries/0_stateless/00754_alter_modify_column_partitions.reference b/tests/queries/0_stateless/00754_alter_modify_column_partitions.reference index a1493508b61..900a3200467 100644 --- a/tests/queries/0_stateless/00754_alter_modify_column_partitions.reference +++ b/tests/queries/0_stateless/00754_alter_modify_column_partitions.reference @@ -1,5 +1,5 @@ *** Check SHOW CREATE TABLE *** -CREATE TABLE default.alter_column\n(\n `x` UInt32, \n `y` Int32\n)\nENGINE = MergeTree\nPARTITION BY x\nORDER BY x\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_column\n(\n `x` UInt32,\n `y` Int32\n)\nENGINE = MergeTree\nPARTITION BY x\nORDER BY x\nSETTINGS index_granularity = 8192 *** Check parts *** 0 0 10 -10 @@ -52,7 +52,7 @@ CREATE TABLE default.alter_column\n(\n `x` UInt32, \n `y` Int32\n)\nENGINE 8 -8 9 -9 *** Check SHOW CREATE TABLE after ALTER MODIFY *** -CREATE TABLE default.alter_column\n(\n `x` UInt32, \n `y` Int64\n)\nENGINE = MergeTree\nPARTITION BY x\nORDER BY x\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_column\n(\n `x` UInt32,\n `y` Int64\n)\nENGINE = MergeTree\nPARTITION BY x\nORDER BY x\nSETTINGS index_granularity = 8192 *** Check parts after ALTER MODIFY *** 0 0 10 -10 diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by.reference b/tests/queries/0_stateless/00754_alter_modify_order_by.reference index f0dc413a186..0279e5ca11b 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by.reference +++ b/tests/queries/0_stateless/00754_alter_modify_order_by.reference @@ -9,4 +9,4 @@ 1 2 1 30 1 2 4 90 *** Check SHOW CREATE TABLE *** -CREATE TABLE default.summing\n(\n `x` UInt32, \n `y` UInt32, \n `z` UInt32, \n `val` UInt32\n)\nENGINE = SummingMergeTree\nPRIMARY KEY (x, y)\nORDER BY (x, y, -z)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.summing\n(\n `x` UInt32,\n `y` UInt32,\n `z` UInt32,\n `val` UInt32\n)\nENGINE = SummingMergeTree\nPRIMARY KEY (x, y)\nORDER BY (x, y, -z)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference index 938a90a27b4..9303d45ea7d 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.reference @@ -9,6 +9,6 @@ 1 2 1 30 1 2 4 90 *** Check SHOW CREATE TABLE *** -CREATE TABLE test.summing_r2\n(\n `x` UInt32, \n `y` UInt32, \n `z` UInt32, \n `val` UInt32\n)\nENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test/summing\', \'r2\')\nPRIMARY KEY (x, y)\nORDER BY (x, y, -z)\nSETTINGS index_granularity = 8192 +CREATE TABLE test.summing_r2\n(\n `x` UInt32,\n `y` UInt32,\n `z` UInt32,\n `val` UInt32\n)\nENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test/summing\', \'r2\')\nPRIMARY KEY (x, y)\nORDER BY (x, y, -z)\nSETTINGS index_granularity = 8192 *** Check SHOW CREATE TABLE after offline ALTER *** -CREATE TABLE test.summing_r2\n(\n `x` UInt32, \n `y` UInt32, \n `z` UInt32, \n `t` UInt32, \n `val` UInt32\n)\nENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test/summing\', \'r2\')\nPRIMARY KEY (x, y)\nORDER BY (x, y, t * t)\nSETTINGS index_granularity = 8192 +CREATE TABLE test.summing_r2\n(\n `x` UInt32,\n `y` UInt32,\n `z` UInt32,\n `t` UInt32,\n `val` UInt32\n)\nENGINE = ReplicatedSummingMergeTree(\'/clickhouse/tables/test/summing\', \'r2\')\nPRIMARY KEY (x, y)\nORDER BY (x, y, t * t)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference index f778c4f5d90..00556b0f8c9 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference @@ -9,10 +9,10 @@ 10003 274972506.6 9175437371954010821 -CREATE TABLE default.compression_codec_multiple_more_types\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)), \n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)), \n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)), \n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.compression_codec_multiple_more_types\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)),\n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)),\n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)),\n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1.5555555555555 hello world! [77] ['John'] 7.1000000000000 xxxxxxxxxxxx [127] ['Henry'] ! 222 !ZSTD -CREATE TABLE default.test_default_delta\n(\n `id` UInt64 CODEC(Delta(8)), \n `data` String CODEC(Delta(1)), \n `somedate` Date CODEC(Delta(2)), \n `somenum` Float64 CODEC(Delta(8)), \n `somestr` FixedString(3) CODEC(Delta(1)), \n `othernum` Int64 CODEC(Delta(8)), \n `yetothernum` Float32 CODEC(Delta(4)), \n `ddd.age` Array(UInt8) CODEC(Delta(1)), \n `ddd.Name` Array(String) CODEC(Delta(1)), \n `ddd.OName` Array(String) CODEC(Delta(1)), \n `ddd.BName` Array(String) CODEC(Delta(1))\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.test_default_delta\n(\n `id` UInt64 CODEC(Delta(8)),\n `data` String CODEC(Delta(1)),\n `somedate` Date CODEC(Delta(2)),\n `somenum` Float64 CODEC(Delta(8)),\n `somestr` FixedString(3) CODEC(Delta(1)),\n `othernum` Int64 CODEC(Delta(8)),\n `yetothernum` Float32 CODEC(Delta(4)),\n `ddd.age` Array(UInt8) CODEC(Delta(1)),\n `ddd.Name` Array(String) CODEC(Delta(1)),\n `ddd.OName` Array(String) CODEC(Delta(1)),\n `ddd.BName` Array(String) CODEC(Delta(1))\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference index b33535364e5..113e413bfac 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference +++ b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference @@ -1,9 +1,9 @@ -CREATE TABLE default.compression_codec_log\n(\n `id` UInt64 CODEC(LZ4), \n `data` String CODEC(ZSTD(1)), \n `ddd` Date CODEC(NONE), \n `somenum` Float64 CODEC(ZSTD(2)), \n `somestr` FixedString(3) CODEC(LZ4HC(7)), \n `othernum` Int64 CODEC(Delta(8))\n)\nENGINE = Log() +CREATE TABLE default.compression_codec_log\n(\n `id` UInt64 CODEC(LZ4),\n `data` String CODEC(ZSTD(1)),\n `ddd` Date CODEC(NONE),\n `somenum` Float64 CODEC(ZSTD(2)),\n `somestr` FixedString(3) CODEC(LZ4HC(7)),\n `othernum` Int64 CODEC(Delta(8))\n)\nENGINE = Log() 1 hello 2018-12-14 1.1 aaa 5 2 world 2018-12-15 2.2 bbb 6 3 ! 2018-12-16 3.3 ccc 7 2 -CREATE TABLE default.compression_codec_multiple_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)), \n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)), \n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)), \n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))\n)\nENGINE = Log() +CREATE TABLE default.compression_codec_multiple_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)),\n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)),\n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)),\n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))\n)\nENGINE = Log() 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 3 buy 2018-10-11 3.3 @@ -11,12 +11,12 @@ CREATE TABLE default.compression_codec_multiple_log\n(\n `id` UInt64 CODEC(LZ 10003 274972506.6 9175437371954010821 -CREATE TABLE default.compression_codec_tiny_log\n(\n `id` UInt64 CODEC(LZ4), \n `data` String CODEC(ZSTD(1)), \n `ddd` Date CODEC(NONE), \n `somenum` Float64 CODEC(ZSTD(2)), \n `somestr` FixedString(3) CODEC(LZ4HC(7)), \n `othernum` Int64 CODEC(Delta(8))\n)\nENGINE = TinyLog() +CREATE TABLE default.compression_codec_tiny_log\n(\n `id` UInt64 CODEC(LZ4),\n `data` String CODEC(ZSTD(1)),\n `ddd` Date CODEC(NONE),\n `somenum` Float64 CODEC(ZSTD(2)),\n `somestr` FixedString(3) CODEC(LZ4HC(7)),\n `othernum` Int64 CODEC(Delta(8))\n)\nENGINE = TinyLog() 1 hello 2018-12-14 1.1 aaa 5 2 world 2018-12-15 2.2 bbb 6 3 ! 2018-12-16 3.3 ccc 7 2 -CREATE TABLE default.compression_codec_multiple_tiny_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)), \n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)), \n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)), \n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))\n)\nENGINE = TinyLog() +CREATE TABLE default.compression_codec_multiple_tiny_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)),\n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)),\n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)),\n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))\n)\nENGINE = TinyLog() 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 3 buy 2018-10-11 3.3 diff --git a/tests/queries/0_stateless/00836_indices_alter.reference b/tests/queries/0_stateless/00836_indices_alter.reference index 6efa25f47b7..7fd63a45d31 100644 --- a/tests/queries/0_stateless/00836_indices_alter.reference +++ b/tests/queries/0_stateless/00836_indices_alter.reference @@ -1,4 +1,4 @@ -CREATE TABLE default.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 1 2 @@ -6,15 +6,15 @@ CREATE TABLE default.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n I 1 2 1 2 1 2 -CREATE TABLE default.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 1 2 1 2 1 2 1 2 -CREATE TABLE default.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE default.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 1 2 @@ -23,6 +23,6 @@ CREATE TABLE default.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n I 1 2 1 2 1 2 -CREATE TABLE default.minmax_idx2\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.minmax_idx2\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 diff --git a/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference b/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference index ec9de160fcc..ce03d1e7de6 100644 --- a/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.reference @@ -1,5 +1,5 @@ -CREATE TABLE test.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 2 1 2 @@ -14,8 +14,8 @@ CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32, \n IN 3 2 19 9 65 75 -CREATE TABLE test.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx3 u64 - i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 + i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 4 1 5 @@ -28,10 +28,10 @@ CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32, \n IN 3 2 19 9 65 75 -CREATE TABLE test.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter1\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 4 1 5 @@ -44,14 +44,14 @@ CREATE TABLE test.minmax_idx_r\n(\n `u64` UInt64, \n `i32` Int32, \n IN 3 2 19 9 65 75 -CREATE TABLE test.minmax_idx2\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx2_r\n(\n `u64` UInt64, \n `i32` Int32, \n INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10, \n INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx2\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx2_r\n(\n `u64` UInt64,\n `i32` Int32,\n INDEX idx1 u64 + i32 TYPE minmax GRANULARITY 10,\n INDEX idx2 u64 * i32 TYPE minmax GRANULARITY 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 3 1 2 1 3 -CREATE TABLE test.minmax_idx2\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE test.minmax_idx2_r\n(\n `u64` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx2\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r1\')\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test.minmax_idx2_r\n(\n `u64` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/indices_alter2\', \'r2\')\nORDER BY u64\nSETTINGS index_granularity = 8192 1 2 1 3 1 2 diff --git a/tests/queries/0_stateless/00933_alter_ttl.reference b/tests/queries/0_stateless/00933_alter_ttl.reference index 9b5cec0f773..545f5644e94 100644 --- a/tests/queries/0_stateless/00933_alter_ttl.reference +++ b/tests/queries/0_stateless/00933_alter_ttl.reference @@ -1,4 +1,4 @@ -CREATE TABLE default.ttl\n(\n `d` Date, \n `a` Int32\n)\nENGINE = MergeTree\nPARTITION BY toDayOfMonth(d)\nORDER BY a\nTTL d + toIntervalDay(1)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl\n(\n `d` Date,\n `a` Int32\n)\nENGINE = MergeTree\nPARTITION BY toDayOfMonth(d)\nORDER BY a\nTTL d + toIntervalDay(1)\nSETTINGS index_granularity = 8192 2100-10-10 3 2100-10-10 4 d Date diff --git a/tests/queries/0_stateless/00933_ttl_simple.reference b/tests/queries/0_stateless/00933_ttl_simple.reference index 102639947a3..a4ef8033328 100644 --- a/tests/queries/0_stateless/00933_ttl_simple.reference +++ b/tests/queries/0_stateless/00933_ttl_simple.reference @@ -6,11 +6,11 @@ 2000-10-10 00:00:00 0 2100-10-10 00:00:00 3 2100-10-10 2 -CREATE TABLE default.ttl_00933_1\n(\n `b` Int32, \n `a` Int32 TTL now() - 1000\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL now() - 1000\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1 0 -CREATE TABLE default.ttl_00933_1\n(\n `b` Int32, \n `a` Int32 TTL now() + 1000\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL now() + 1000\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1 1 -CREATE TABLE default.ttl_00933_1\n(\n `b` Int32, \n `a` Int32 TTL today() - 1\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL today() - 1\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1 0 -CREATE TABLE default.ttl_00933_1\n(\n `b` Int32, \n `a` Int32 TTL today() + 1\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL today() + 1\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1 1 diff --git a/tests/queries/0_stateless/00980_merge_alter_settings.reference b/tests/queries/0_stateless/00980_merge_alter_settings.reference index 340cf29ce89..7f8aa23b722 100644 --- a/tests/queries/0_stateless/00980_merge_alter_settings.reference +++ b/tests/queries/0_stateless/00980_merge_alter_settings.reference @@ -1,6 +1,6 @@ -CREATE TABLE default.table_for_alter\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 1, parts_to_delay_insert = 1 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100 2 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64, \n `Data` String, \n `Data2` UInt64\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 15 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = MergeTree()\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 15 diff --git a/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference b/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference index ab006ea6931..2682051751b 100644 --- a/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference +++ b/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.reference @@ -1,12 +1,12 @@ -CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 -CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192 4 4 4 4 6 6 -CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1 -CREATE TABLE default.replicated_table_for_alter2\n(\n `id` UInt64, \n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 -CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64, \n `Data` String, \n `Data2` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1, check_delay_period = 15 -CREATE TABLE default.replicated_table_for_alter2\n(\n `id` UInt64, \n `Data` String, \n `Data2` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1 +CREATE TABLE default.replicated_table_for_alter2\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.replicated_table_for_alter1\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'1\')\nORDER BY id\nSETTINGS index_granularity = 8192, use_minimalistic_part_header_in_zookeeper = 1, check_delay_period = 15 +CREATE TABLE default.replicated_table_for_alter2\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/replicated_table_for_alter\', \'2\')\nORDER BY id\nSETTINGS index_granularity = 8192, parts_to_throw_insert = 1, parts_to_delay_insert = 1 diff --git a/tests/queries/0_stateless/00998_constraints_all_tables.reference b/tests/queries/0_stateless/00998_constraints_all_tables.reference index 3de251daa71..0ec8b004ae4 100644 --- a/tests/queries/0_stateless/00998_constraints_all_tables.reference +++ b/tests/queries/0_stateless/00998_constraints_all_tables.reference @@ -10,5 +10,5 @@ 0 0 3 -CREATE TABLE default.constrained\n(\n `URL` String, \n CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = \'yandex.ru\', \n CONSTRAINT is_utf8 CHECK isValidUTF8(URL)\n)\nENGINE = Log -CREATE TABLE default.constrained2\n(\n `URL` String, \n CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = \'yandex.ru\', \n CONSTRAINT is_utf8 CHECK isValidUTF8(URL)\n)\nENGINE = Log +CREATE TABLE default.constrained\n(\n `URL` String,\n CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = \'yandex.ru\',\n CONSTRAINT is_utf8 CHECK isValidUTF8(URL)\n)\nENGINE = Log +CREATE TABLE default.constrained2\n(\n `URL` String,\n CONSTRAINT is_yandex CHECK domainWithoutWWW(URL) = \'yandex.ru\',\n CONSTRAINT is_utf8 CHECK isValidUTF8(URL)\n)\nENGINE = Log diff --git a/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference b/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference index c2d7d849fae..548952c3a6a 100644 --- a/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference +++ b/tests/queries/0_stateless/01033_storage_odbc_parsing_exception_check.reference @@ -1 +1 @@ -CREATE TABLE default.BannerDict\n(\n `BannerID` UInt64, \n `CompaignID` UInt64\n)\nENGINE = ODBC(\'DSN=pgconn;Database=postgres\', \'somedb\', \'bannerdict\') +CREATE TABLE default.BannerDict\n(\n `BannerID` UInt64,\n `CompaignID` UInt64\n)\nENGINE = ODBC(\'DSN=pgconn;Database=postgres\', \'somedb\', \'bannerdict\') diff --git a/tests/queries/0_stateless/01055_compact_parts_1.reference b/tests/queries/0_stateless/01055_compact_parts_1.reference index b99f336d3b0..c5311a0b479 100644 --- a/tests/queries/0_stateless/01055_compact_parts_1.reference +++ b/tests/queries/0_stateless/01055_compact_parts_1.reference @@ -1,2 +1,2 @@ -CREATE TABLE default.mt_compact\n(\n `a` Int32, \n `s` String\n)\nENGINE = MergeTree\nPARTITION BY a\nORDER BY a\nSETTINGS index_granularity_bytes = 0, index_granularity = 8192 -CREATE TABLE default.mt_compact\n(\n `a` Int32, \n `s` String\n)\nENGINE = MergeTree\nPARTITION BY a\nORDER BY a\nSETTINGS index_granularity_bytes = 0, min_rows_for_wide_part = 0, index_granularity = 8192, parts_to_delay_insert = 300 +CREATE TABLE default.mt_compact\n(\n `a` Int32,\n `s` String\n)\nENGINE = MergeTree\nPARTITION BY a\nORDER BY a\nSETTINGS index_granularity_bytes = 0, index_granularity = 8192 +CREATE TABLE default.mt_compact\n(\n `a` Int32,\n `s` String\n)\nENGINE = MergeTree\nPARTITION BY a\nORDER BY a\nSETTINGS index_granularity_bytes = 0, min_rows_for_wide_part = 0, index_granularity = 8192, parts_to_delay_insert = 300 diff --git a/tests/queries/0_stateless/01069_database_memory.reference b/tests/queries/0_stateless/01069_database_memory.reference index e7486d57276..cfccf5b1757 100644 --- a/tests/queries/0_stateless/01069_database_memory.reference +++ b/tests/queries/0_stateless/01069_database_memory.reference @@ -5,4 +5,4 @@ CREATE DATABASE memory_01069\nENGINE = Memory() 4 3 4 -CREATE TABLE memory_01069.file\n(\n `n` UInt8\n)\nENGINE = File(\'CSV\') +CREATE TABLE memory_01069.file\n(`n` UInt8\n)\nENGINE = File(\'CSV\') diff --git a/tests/queries/0_stateless/01070_alter_with_ttl.reference b/tests/queries/0_stateless/01070_alter_with_ttl.reference index de7833472a1..202ac2ac10f 100644 --- a/tests/queries/0_stateless/01070_alter_with_ttl.reference +++ b/tests/queries/0_stateless/01070_alter_with_ttl.reference @@ -1,2 +1,2 @@ -CREATE TABLE default.alter_ttl\n(\n `i` Int32, \n `s` String TTL toDate(\'2020-01-01\')\n)\nENGINE = MergeTree\nORDER BY i\nTTL toDate(\'2020-05-05\')\nSETTINGS index_granularity = 8192 -CREATE TABLE default.alter_ttl\n(\n `d` Date, \n `s` String TTL d + toIntervalDay(1)\n)\nENGINE = MergeTree\nORDER BY d\nTTL d + toIntervalMonth(1)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_ttl\n(\n `i` Int32,\n `s` String TTL toDate(\'2020-01-01\')\n)\nENGINE = MergeTree\nORDER BY i\nTTL toDate(\'2020-05-05\')\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_ttl\n(\n `d` Date,\n `s` String TTL d + toIntervalDay(1)\n)\nENGINE = MergeTree\nORDER BY d\nTTL d + toIntervalMonth(1)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01079_alter_default_zookeeper.reference b/tests/queries/0_stateless/01079_alter_default_zookeeper.reference index 62d26bc9b4b..758150a7799 100644 --- a/tests/queries/0_stateless/01079_alter_default_zookeeper.reference +++ b/tests/queries/0_stateless/01079_alter_default_zookeeper.reference @@ -1,11 +1,11 @@ -CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` String DEFAULT \'10\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date,\n `key` UInt64,\n `value` String DEFAULT \'10\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 1000 -CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt64 DEFAULT \'10\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt64 DEFAULT 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date,\n `key` UInt64,\n `value` UInt64 DEFAULT \'10\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date,\n `key` UInt64,\n `value` UInt64 DEFAULT 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 1000 -CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt64 DEFAULT 100\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt16 DEFAULT 100\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date,\n `key` UInt64,\n `value` UInt64 DEFAULT 100\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date,\n `key` UInt64,\n `value` UInt16 DEFAULT 100\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 10000 -CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt8 DEFAULT 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt8 DEFAULT 10, \n `better_column` UInt8 DEFAULT \'1\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.alter_default\n(\n `date` Date, \n `key` UInt64, \n `value` UInt8 DEFAULT 10, \n `better_column` UInt8 DEFAULT \'1\', \n `other_date` String DEFAULT 1\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date,\n `key` UInt64,\n `value` UInt8 DEFAULT 10\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date,\n `key` UInt64,\n `value` UInt8 DEFAULT 10,\n `better_column` UInt8 DEFAULT \'1\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_default\n(\n `date` Date,\n `key` UInt64,\n `value` UInt8 DEFAULT 10,\n `better_column` UInt8 DEFAULT \'1\',\n `other_date` String DEFAULT 1\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/alter_default\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference b/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference index ea3fbec34a8..deb26676f39 100644 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference @@ -1,6 +1,6 @@ Wrong column name. -CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64, \n `value1` UInt8, \n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64, \n `value1` UInt8, \n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 syntax error at begin of string. 7 Hello diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference index 2007eda0f07..138f09f2634 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference @@ -1,11 +1,11 @@ -CREATE TABLE test_01083.file\n(\n `n` Int8\n)\nENGINE = File(\'TSVWithNamesAndTypes\') -CREATE TABLE test_01083.buffer\n(\n `n` Int8\n)\nENGINE = Buffer(\'test_01083\', \'file\', 16, 10, 200, 10000, 1000000, 10000000, 1000000000) -CREATE TABLE test_01083.merge\n(\n `n` Int8\n)\nENGINE = Merge(\'test_01083\', \'distributed\') +CREATE TABLE test_01083.file\n(`n` Int8\n)\nENGINE = File(\'TSVWithNamesAndTypes\') +CREATE TABLE test_01083.buffer\n(`n` Int8\n)\nENGINE = Buffer(\'test_01083\', \'file\', 16, 10, 200, 10000, 1000000, 10000000, 1000000000) +CREATE TABLE test_01083.merge\n(`n` Int8\n)\nENGINE = Merge(\'test_01083\', \'distributed\') CREATE TABLE test_01083.merge_tf AS merge(\'test_01083\', \'.*\') -CREATE TABLE test_01083.distributed\n(\n `n` Int8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'test_01083\', \'file\') +CREATE TABLE test_01083.distributed\n(`n` Int8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'test_01083\', \'file\') CREATE TABLE test_01083.distributed_tf AS cluster(\'test_shard_localhost\', \'test_01083\', \'buffer\') -CREATE TABLE test_01083.url\n(\n `n` UInt64, \n `col` String\n)\nENGINE = URL(\'https://localhost:8443/?query=select+n,+_table+from+test_01083.merge+format+CSV\', \'CSV\') +CREATE TABLE test_01083.url\n(\n `n` UInt64,\n `col` String\n)\nENGINE = URL(\'https://localhost:8443/?query=select+n,+_table+from+test_01083.merge+format+CSV\', \'CSV\') CREATE TABLE test_01083.rich_syntax AS remote(\'localhos{x|y|t}\', cluster(\'test_shard_localhost\', remote(\'127.0.0.{1..4}\', \'test_01083\', \'view\'))) -CREATE VIEW test_01083.view\n(\n `n` Int64\n) AS\nSELECT toInt64(n) AS n\nFROM \n(\n SELECT toString(n) AS n\n FROM test_01083.merge\n WHERE _table != \'qwerty\'\n ORDER BY _table ASC\n)\nUNION ALL\nSELECT *\nFROM test_01083.file -CREATE DICTIONARY test_01083.dict\n(\n `n` UInt64, \n `col` String DEFAULT \'42\'\n)\nPRIMARY KEY n\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9440 SECURE 1 USER \'default\' TABLE \'url\' DB \'test_01083\'))\nLIFETIME(MIN 0 MAX 1)\nLAYOUT(CACHE(SIZE_IN_CELLS 1)) +CREATE VIEW test_01083.view\n(`n` Int64\n) AS\nSELECT toInt64(n) AS n\nFROM \n(\n SELECT toString(n) AS n\n FROM test_01083.merge\n WHERE _table != \'qwerty\'\n ORDER BY _table ASC\n)\nUNION ALL\nSELECT *\nFROM test_01083.file +CREATE DICTIONARY test_01083.dict\n(\n \n `n` UInt64,\n \n `col` String DEFAULT \'42\'\n)\nPRIMARY KEY n\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9440 SECURE 1 USER \'default\' TABLE \'url\' DB \'test_01083\'))\nLIFETIME(MIN 0 MAX 1)\nLAYOUT(CACHE(SIZE_IN_CELLS 1)) 16 diff --git a/tests/queries/0_stateless/01135_default_and_alter_zookeeper.reference b/tests/queries/0_stateless/01135_default_and_alter_zookeeper.reference index f7c4a48b4bc..6a5dd7223bd 100644 --- a/tests/queries/0_stateless/01135_default_and_alter_zookeeper.reference +++ b/tests/queries/0_stateless/01135_default_and_alter_zookeeper.reference @@ -1,2 +1,2 @@ 4 -CREATE TABLE default.default_table\n(\n `id` UInt64, \n `enum_column` Enum8(\'undefined\' = 0, \'fox\' = 1, \'index\' = 2) DEFAULT \'fox\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/default_table\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.default_table\n(\n `id` UInt64,\n `enum_column` Enum8(\'undefined\' = 0, \'fox\' = 1, \'index\' = 2) DEFAULT \'fox\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/default_table\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference index e2d6007c57f..5457becfeda 100644 --- a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference +++ b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference @@ -1,6 +1,6 @@ 1 -CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date, \n `key` UInt64, \n `renamed_value1` String, \n `value2` String, \n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date,\n `key` UInt64,\n `value1` String,\n `value2` String,\n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date,\n `key` UInt64,\n `renamed_value1` String,\n `value2` String,\n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 1 date key renamed_value1 value2 value3 2019-10-02 1 1 1 1 diff --git a/tests/queries/0_stateless/01213_alter_rename_nested.reference b/tests/queries/0_stateless/01213_alter_rename_nested.reference index 2641df46aeb..403e87256fe 100644 --- a/tests/queries/0_stateless/01213_alter_rename_nested.reference +++ b/tests/queries/0_stateless/01213_alter_rename_nested.reference @@ -1,10 +1,10 @@ [8,9,10] ['a','b','c'] -CREATE TABLE default.table_for_rename_nested\n(\n `date` Date, \n `key` UInt64, \n `n.x` Array(UInt32), \n `n.y` Array(String), \n `value1` Array(Array(LowCardinality(String)))\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_rename_nested\n(\n `date` Date, \n `key` UInt64, \n `n.renamed_x` Array(UInt32), \n `n.renamed_y` Array(String), \n `value1` Array(Array(LowCardinality(String)))\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_nested\n(\n `date` Date,\n `key` UInt64,\n `n.x` Array(UInt32),\n `n.y` Array(String),\n `value1` Array(Array(LowCardinality(String)))\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_nested\n(\n `date` Date,\n `key` UInt64,\n `n.renamed_x` Array(UInt32),\n `n.renamed_y` Array(String),\n `value1` Array(Array(LowCardinality(String)))\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 7 [8,9,10] 7 ['a','b','c'] [['7']] -CREATE TABLE default.table_for_rename_nested\n(\n `date` Date, \n `key` UInt64, \n `n.renamed_x` Array(UInt32), \n `n.renamed_y` Array(String), \n `renamed_value1` Array(Array(LowCardinality(String)))\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_nested\n(\n `date` Date,\n `key` UInt64,\n `n.renamed_x` Array(UInt32),\n `n.renamed_y` Array(String),\n `renamed_value1` Array(Array(LowCardinality(String)))\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 date key n.renamed_x n.renamed_y renamed_value1 2019-10-01 7 [8,9,10] ['a','b','c'] [['7']] diff --git a/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper.reference b/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper.reference index 251e664b522..a4759ecb0f7 100644 --- a/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper.reference +++ b/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper.reference @@ -1,17 +1,17 @@ date key value1 value2 2019-10-02 1 1 Hello 1 -CREATE TABLE default.table_rename_with_default\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String DEFAULT concat(\'Hello \', value1), \n `value3` String ALIAS concat(\'Word \', value1)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_rename_with_default\n(\n `date` Date,\n `key` UInt64,\n `value1` String,\n `value2` String DEFAULT concat(\'Hello \', value1),\n `value3` String ALIAS concat(\'Word \', value1)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 date key renamed_value1 value2 2019-10-02 1 1 Hello 1 -CREATE TABLE default.table_rename_with_default\n(\n `date` Date, \n `key` UInt64, \n `renamed_value1` String, \n `value2` String DEFAULT concat(\'Hello \', renamed_value1), \n `value3` String ALIAS concat(\'Word \', renamed_value1)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_rename_with_default\n(\n `date` Date,\n `key` UInt64,\n `renamed_value1` String,\n `value2` String DEFAULT concat(\'Hello \', renamed_value1),\n `value3` String ALIAS concat(\'Word \', renamed_value1)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 Hello 1 Word 1 date1 date2 value1 value2 2019-10-02 2018-10-02 1 1 -CREATE TABLE default.table_rename_with_ttl\n(\n `date1` Date, \n `date2` Date, \n `value1` String, \n `value2` String TTL date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_rename_with_ttl\n(\n `date1` Date,\n `date2` Date,\n `value1` String,\n `value2` String TTL date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192 renamed_date1 date2 value1 value2 2019-10-02 2018-10-02 1 1 -CREATE TABLE default.table_rename_with_ttl\n(\n `renamed_date1` Date, \n `date2` Date, \n `value1` String, \n `value2` String TTL renamed_date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_rename_with_ttl\n(\n `renamed_date1` Date,\n `date2` Date,\n `value1` String,\n `value2` String TTL renamed_date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192 renamed_date1 renamed_date2 value1 value2 2019-10-02 2018-10-02 1 1 -CREATE TABLE default.table_rename_with_ttl\n(\n `renamed_date1` Date, \n `renamed_date2` Date, \n `value1` String, \n `value2` String TTL renamed_date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL renamed_date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_rename_with_ttl\n(\n `renamed_date1` Date,\n `renamed_date2` Date,\n `value1` String,\n `value2` String TTL renamed_date1 + toIntervalMonth(10000)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/table_rename_with_ttl\', \'1\')\nORDER BY tuple()\nTTL renamed_date2 + toIntervalMonth(10000)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01213_alter_table_rename_nested.reference b/tests/queries/0_stateless/01213_alter_table_rename_nested.reference index 8e6d93dbcce..1b89cf8f461 100644 --- a/tests/queries/0_stateless/01213_alter_table_rename_nested.reference +++ b/tests/queries/0_stateless/01213_alter_table_rename_nested.reference @@ -1,6 +1,6 @@ [8,9,10] ['a','b','c'] -CREATE TABLE default.table_for_rename_nested\n(\n `date` Date, \n `key` UInt64, \n `n.x` Array(UInt32), \n `n.y` Array(String), \n `value1` String\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_rename_nested\n(\n `date` Date, \n `key` UInt64, \n `n.renamed_x` Array(UInt32), \n `n.renamed_y` Array(String), \n `value1` String\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_nested\n(\n `date` Date,\n `key` UInt64,\n `n.x` Array(UInt32),\n `n.y` Array(String),\n `value1` String\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_nested\n(\n `date` Date,\n `key` UInt64,\n `n.renamed_x` Array(UInt32),\n `n.renamed_y` Array(String),\n `value1` String\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 7 [8,9,10] 7 ['a','b','c'] diff --git a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference index 524fbdd26fc..96d4393e06b 100644 --- a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference +++ b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference @@ -12,7 +12,7 @@ LAYOUT(FLAT()) NOT_LOADED CREATE TABLE dict_db_01224_dictionary.`dict_db_01224.dict` ( - `key` UInt64, + `key` UInt64, `val` UInt64 ) ENGINE = Dictionary(`dict_db_01224.dict`) diff --git a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.reference b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.reference index 14ddc093143..3363df5fb98 100644 --- a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.reference +++ b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.reference @@ -1,6 +1,6 @@ CREATE TABLE dict_db_01225_dictionary.`dict_db_01225.dict` ( - `key` UInt64, + `key` UInt64, `val` UInt64 ) ENGINE = Dictionary(`dict_db_01225.dict`) diff --git a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference index 04ae001675f..dfff8c7be00 100644 --- a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference +++ b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference @@ -1,3 +1,3 @@ -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64, \n `i32` Int32, \n `f64` Float64, \n `d` Decimal(10, 2), \n `s` String, \n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), \n `dt` Date, \n INDEX bloom_filter_a i32 TYPE bloom_filter(0.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64, \n `i32` Int32, \n `f64` Float64, \n `d` Decimal(10, 2), \n `s` String, \n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), \n `dt` Date, \n INDEX bloom_filter_a i32 TYPE bloom_filter(0.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64, \n `i32` Int32, \n `f64` Float64, \n `d` Decimal(10, 2), \n `s` String, \n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), \n `dt` Date, \n INDEX bloom_filter_a i32 TYPE bloom_filter(1.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(0.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(0.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(1.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.reference b/tests/queries/0_stateless/01272_suspicious_codecs.reference index 559b6df2693..de91a1ddb25 100644 --- a/tests/queries/0_stateless/01272_suspicious_codecs.reference +++ b/tests/queries/0_stateless/01272_suspicious_codecs.reference @@ -1,16 +1,16 @@ -CREATE TABLE default.codecs1\n(\n `a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs2\n(\n `a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs3\n(\n `a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs4\n(\n `a` UInt8 CODEC(LZ4, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs5\n(\n `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs6\n(\n `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs7\n(\n `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs8\n(\n `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs1\n(\n `a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs2\n(\n `a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs3\n(\n `a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs4\n(\n `a` UInt8 CODEC(LZ4, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs5\n(\n `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs6\n(\n `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs7\n(\n `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs8\n(\n `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs1\n(`a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs2\n(`a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs3\n(`a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs4\n(`a` UInt8 CODEC(LZ4, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs5\n(`a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs6\n(`a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs7\n(`a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs8\n(`a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs1\n(`a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs2\n(`a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs3\n(`a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs4\n(`a` UInt8 CODEC(LZ4, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs5\n(`a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs6\n(`a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs7\n(`a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs8\n(`a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference b/tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference index d81601b92c5..e1ea5a778da 100644 --- a/tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference +++ b/tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference @@ -7,7 +7,7 @@ 2019-10-01 6 6 7 6 + 7 2019-10-02 7 7 8 7 + 8 2019-10-03 8 8 9 8 + 9 -CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value4` String, \n `value5` String, \n `value3` String DEFAULT concat(value4, \' + \', value5)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename\n(\n `date` Date,\n `key` UInt64,\n `value4` String,\n `value5` String,\n `value3` String DEFAULT concat(value4, \' + \', value5)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 2019-10-01 0 0 1 0 + 1 2019-10-02 1 1 2 1 + 2 2019-10-03 2 2 3 2 + 3 @@ -36,7 +36,7 @@ CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \ 2019-10-03 17 17 18 17 + 18 2019-10-01 18 18 19 18 + 19 2019-10-02 19 19 20 19 + 20 -CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String DEFAULT concat(value1, \' + \', value2)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename\n(\n `date` Date,\n `key` UInt64,\n `value1` String,\n `value2` String,\n `value3` String DEFAULT concat(value1, \' + \', value2)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 2019-10-01 0 0 1 0 + 1 2019-10-02 1 1 2 1 + 2 2019-10-03 2 2 3 2 + 3 diff --git a/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference b/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference index 5d721230db3..c430b6a28af 100644 --- a/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference +++ b/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference @@ -7,7 +7,7 @@ 2019-10-01 6 6 7 2019-10-02 7 7 8 2019-10-03 8 8 9 -CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value4` String, \n `value5` String, \n `value3` String MATERIALIZED concat(value4, \' + \', value5)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename\n(\n `date` Date,\n `key` UInt64,\n `value4` String,\n `value5` String,\n `value3` String MATERIALIZED concat(value4, \' + \', value5)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 2019-10-01 0 0 1 2019-10-02 1 1 2 2019-10-03 2 2 3 @@ -38,7 +38,7 @@ CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \ 2019-10-01 18 18 19 2019-10-02 19 19 20 -- rename columns back -- -CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String MATERIALIZED concat(value1, \' + \', value2)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename\n(\n `date` Date,\n `key` UInt64,\n `value1` String,\n `value2` String,\n `value3` String MATERIALIZED concat(value1, \' + \', value2)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 2019-10-01 0 0 1 2019-10-02 1 1 2 2019-10-03 2 2 3 diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint.reference b/tests/queries/0_stateless/01277_alter_rename_column_constraint.reference index cb1842f95da..4316c7fa1b9 100644 --- a/tests/queries/0_stateless/01277_alter_rename_column_constraint.reference +++ b/tests/queries/0_stateless/01277_alter_rename_column_constraint.reference @@ -7,7 +7,7 @@ 2019-10-01 6 6 7 8 2019-10-02 7 7 8 9 2019-10-03 8 8 9 10 -CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value4` String, \n `value5` String, \n `value3` String, \n CONSTRAINT cs_value1 CHECK toInt64(value4) < toInt64(value5), \n CONSTRAINT cs_value2 CHECK toInt64(value5) < toInt64(value3)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename\n(\n `date` Date,\n `key` UInt64,\n `value4` String,\n `value5` String,\n `value3` String,\n CONSTRAINT cs_value1 CHECK toInt64(value4) < toInt64(value5),\n CONSTRAINT cs_value2 CHECK toInt64(value5) < toInt64(value3)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 2019-10-01 0 0 1 2 2019-10-02 1 1 2 3 2019-10-03 2 2 3 4 @@ -38,7 +38,7 @@ CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \ 2019-10-01 18 18 19 20 2019-10-02 19 19 20 21 -- rename columns back -- -CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String, \n CONSTRAINT cs_value1 CHECK toInt64(value1) < toInt64(value2), \n CONSTRAINT cs_value2 CHECK toInt64(value2) < toInt64(value3)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename\n(\n `date` Date,\n `key` UInt64,\n `value1` String,\n `value2` String,\n `value3` String,\n CONSTRAINT cs_value1 CHECK toInt64(value1) < toInt64(value2),\n CONSTRAINT cs_value2 CHECK toInt64(value2) < toInt64(value3)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 2019-10-01 0 0 1 2 2019-10-02 1 1 2 3 2019-10-03 2 2 3 4 diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.reference b/tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.reference index 9ca17dbbc0a..6a9d5a3bdcf 100644 --- a/tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.reference +++ b/tests/queries/0_stateless/01277_alter_rename_column_constraint_zookeeper.reference @@ -7,7 +7,7 @@ 2019-10-01 6 6 7 8 2019-10-02 7 7 8 9 2019-10-03 8 8 9 10 -CREATE TABLE default.table_for_rename1\n(\n `date` Date, \n `key` UInt64, \n `value4` String, \n `value5` String, \n `value3` String, \n CONSTRAINT cs_value1 CHECK toInt64(value4) < toInt64(value5), \n CONSTRAINT cs_value2 CHECK toInt64(value5) < toInt64(value3)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_for_rename\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename1\n(\n `date` Date,\n `key` UInt64,\n `value4` String,\n `value5` String,\n `value3` String,\n CONSTRAINT cs_value1 CHECK toInt64(value4) < toInt64(value5),\n CONSTRAINT cs_value2 CHECK toInt64(value5) < toInt64(value3)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_for_rename\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 2019-10-01 0 0 1 2 2019-10-02 1 1 2 3 2019-10-03 2 2 3 4 @@ -38,7 +38,7 @@ CREATE TABLE default.table_for_rename1\n(\n `date` Date, \n `key` UInt64, 2019-10-01 18 18 19 20 2019-10-02 19 19 20 21 -- rename columns back -- -CREATE TABLE default.table_for_rename1\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String, \n CONSTRAINT cs_value1 CHECK toInt64(value1) < toInt64(value2), \n CONSTRAINT cs_value2 CHECK toInt64(value2) < toInt64(value3)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_for_rename\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename1\n(\n `date` Date,\n `key` UInt64,\n `value1` String,\n `value2` String,\n `value3` String,\n CONSTRAINT cs_value1 CHECK toInt64(value1) < toInt64(value2),\n CONSTRAINT cs_value2 CHECK toInt64(value2) < toInt64(value3)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_for_rename\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 2019-10-01 0 0 1 2 2019-10-02 1 1 2 3 2019-10-03 2 2 3 4 diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.reference b/tests/queries/0_stateless/01278_alter_rename_combination.reference index 3f00378b4b7..cc912e9b265 100644 --- a/tests/queries/0_stateless/01278_alter_rename_combination.reference +++ b/tests/queries/0_stateless/01278_alter_rename_combination.reference @@ -1,15 +1,15 @@ -CREATE TABLE default.rename_table\n(\n `key` Int32, \n `old_value1` Int32, \n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rename_table\n(\n `key` Int32,\n `old_value1` Int32,\n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 key old_value1 value1 1 2 3 -CREATE TABLE default.rename_table\n(\n `k` Int32, \n `v1` Int32, \n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rename_table\n(\n `k` Int32,\n `v1` Int32,\n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 k v1 v2 1 2 3 4 5 6 ---polymorphic--- -CREATE TABLE default.rename_table_polymorphic\n(\n `key` Int32, \n `old_value1` Int32, \n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 10000, index_granularity = 8192 +CREATE TABLE default.rename_table_polymorphic\n(\n `key` Int32,\n `old_value1` Int32,\n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 10000, index_granularity = 8192 key old_value1 value1 1 2 3 -CREATE TABLE default.rename_table_polymorphic\n(\n `k` Int32, \n `v1` Int32, \n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 10000, index_granularity = 8192 +CREATE TABLE default.rename_table_polymorphic\n(\n `k` Int32,\n `v1` Int32,\n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 10000, index_granularity = 8192 k v1 v2 1 2 3 4 5 6 diff --git a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference index f0a906147ac..bf3358aea60 100644 --- a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference +++ b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference @@ -1,23 +1,23 @@ -CREATE TABLE default.rename_table_multiple\n(\n `key` Int32, \n `value1_string` String, \n `value2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 key value1_string value2 1 2 3 -CREATE TABLE default.rename_table_multiple\n(\n `key` Int32, \n `value1_string` String, \n `value2_old` Int32, \n `value2` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int32,\n `value2` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 key value1_string value2_old value2 1 2 3 7 4 5 6 7 -CREATE TABLE default.rename_table_multiple\n(\n `key` Int32, \n `value1_string` String, \n `value2_old` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 key value1_string value2_old 1 2 7 4 5 7 7 8 10 -CREATE TABLE default.rename_table_multiple_compact\n(\n `key` Int32, \n `value1_string` String, \n `value2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 100000, index_granularity = 8192 +CREATE TABLE default.rename_table_multiple_compact\n(\n `key` Int32,\n `value1_string` String,\n `value2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 100000, index_granularity = 8192 key value1_string value2 1 2 3 -CREATE TABLE default.rename_table_multiple_compact\n(\n `key` Int32, \n `value1_string` String, \n `value2_old` Int32, \n `value2` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 100000, index_granularity = 8192 +CREATE TABLE default.rename_table_multiple_compact\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int32,\n `value2` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 100000, index_granularity = 8192 key value1_string value2_old value2 1 2 3 7 4 5 6 7 -CREATE TABLE default.rename_table_multiple_compact\n(\n `key` Int32, \n `value1_string` String, \n `value2_old` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 100000, index_granularity = 8192 +CREATE TABLE default.rename_table_multiple_compact\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 100000, index_granularity = 8192 key value1_string value2_old 1 2 7 4 5 7 From d2052dd1d4b1f6192d9b7283dec50c3ec3736ee7 Mon Sep 17 00:00:00 2001 From: hexiaoting <“hewenting_ict@163.com”> Date: Fri, 5 Jun 2020 17:13:13 +0800 Subject: [PATCH 022/329] show clusters --- .../InterpreterShowTablesQuery.cpp | 27 +++++++++++++++ src/Parsers/ASTShowTablesQuery.cpp | 19 ++++++++++- src/Parsers/ASTShowTablesQuery.h | 5 ++- src/Parsers/ParserShowTablesQuery.cpp | 34 +++++++++++++++++++ src/Parsers/ParserShowTablesQuery.h | 2 +- .../0_stateless/01293_show_create_cluster.sql | 3 ++ 6 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01293_show_create_cluster.sql diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 3660a41c474..a925466f72b 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB @@ -33,6 +34,32 @@ String InterpreterShowTablesQuery::getRewrittenQuery() if (query.databases) return "SELECT name FROM system.databases"; + /// SHOW CLUSTER/CLUSTERS + if (query.clusters) + { + std::stringstream rewritten_query; + rewritten_query << "SELECT cluster FROM system.clusters"; + + if (!query.like.empty()) + { + rewritten_query << " WHERE cluster " << (query.not_like ? "NOT " : "") << "LIKE " << std::quoted(query.like, '\''); + } + + if (query.limit_length) + rewritten_query << " LIMIT " << query.limit_length; + + return rewritten_query.str(); + } + else if (query.cluster) + { + std::stringstream rewritten_query; + rewritten_query << "SELECT * FROM system.clusters"; + + rewritten_query << " WHERE cluster = " << std::quoted(query.cluster_str, '\''); + + return rewritten_query.str(); + } + if (query.temporary && !query.from.empty()) throw Exception("The `FROM` and `TEMPORARY` cannot be used together in `SHOW TABLES`", ErrorCodes::SYNTAX_ERROR); diff --git a/src/Parsers/ASTShowTablesQuery.cpp b/src/Parsers/ASTShowTablesQuery.cpp index 82b773ea70a..39904061200 100644 --- a/src/Parsers/ASTShowTablesQuery.cpp +++ b/src/Parsers/ASTShowTablesQuery.cpp @@ -2,7 +2,6 @@ #include #include - namespace DB { @@ -20,6 +19,24 @@ void ASTShowTablesQuery::formatQueryImpl(const FormatSettings & settings, Format { settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW DATABASES" << (settings.hilite ? hilite_none : ""); } + else if (clusters) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CLUSTERS" << (settings.hilite ? hilite_none : ""); + if (!like.empty()) + settings.ostr << (settings.hilite ? hilite_keyword : "") << (not_like ? " NOT" : "") << " LIKE " << (settings.hilite ? hilite_none : "") + << std::quoted(like, '\''); + + if (limit_length) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " LIMIT " << (settings.hilite ? hilite_none : ""); + limit_length->formatImpl(settings, state, frame); + } + } + else if (cluster) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CLUSTER" << (settings.hilite ? hilite_none : ""); + settings.ostr << " " << backQuoteIfNeed(cluster_str); + } else { settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << (temporary ? "TEMPORARY " : "") << diff --git a/src/Parsers/ASTShowTablesQuery.h b/src/Parsers/ASTShowTablesQuery.h index be0d73a3ac7..f14d6e7bd33 100644 --- a/src/Parsers/ASTShowTablesQuery.h +++ b/src/Parsers/ASTShowTablesQuery.h @@ -9,14 +9,17 @@ namespace DB { -/** Query SHOW TABLES or SHOW DATABASES +/** Query SHOW TABLES or SHOW DATABASES or SHOW CLUSTERS */ class ASTShowTablesQuery : public ASTQueryWithOutput { public: bool databases{false}; + bool clusters{false}; + bool cluster{false}; bool dictionaries{false}; bool temporary{false}; + String cluster_str; String from; String like; bool not_like{false}; diff --git a/src/Parsers/ParserShowTablesQuery.cpp b/src/Parsers/ParserShowTablesQuery.cpp index 36caf24e623..fb29b6d99cd 100644 --- a/src/Parsers/ParserShowTablesQuery.cpp +++ b/src/Parsers/ParserShowTablesQuery.cpp @@ -6,8 +6,10 @@ #include #include #include +#include #include +#include namespace DB @@ -20,6 +22,8 @@ bool ParserShowTablesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserKeyword s_temporary("TEMPORARY"); ParserKeyword s_tables("TABLES"); ParserKeyword s_databases("DATABASES"); + ParserKeyword s_clusters("CLUSTERS"); + ParserKeyword s_cluster("CLUSTER"); ParserKeyword s_dictionaries("DICTIONARIES"); ParserKeyword s_from("FROM"); ParserKeyword s_in("IN"); @@ -43,6 +47,36 @@ bool ParserShowTablesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec { query->databases = true; } + else if (s_clusters.ignore(pos)) + { + query->clusters = true; + + if (s_not.ignore(pos, expected)) + query->not_like = true; + + if (s_like.ignore(pos, expected)) + { + if (!like_p.parse(pos, like, expected)) + return false; + } + else if (query->not_like) + return false; + if (s_limit.ignore(pos, expected)) + { + if (!exp_elem.parse(pos, query->limit_length, expected)) + return false; + } + } + else if (s_cluster.ignore(pos)) + { + query->cluster = true; + + String cluster_str; + if (!parseIdentifierOrStringLiteral(pos, expected, cluster_str)) + return false; + + query->cluster_str = std::move(cluster_str); + } else { if (s_temporary.ignore(pos)) diff --git a/src/Parsers/ParserShowTablesQuery.h b/src/Parsers/ParserShowTablesQuery.h index 1bbd3cb4ef6..4fd11d8e2a0 100644 --- a/src/Parsers/ParserShowTablesQuery.h +++ b/src/Parsers/ParserShowTablesQuery.h @@ -14,7 +14,7 @@ namespace DB class ParserShowTablesQuery : public IParserBase { protected: - const char * getName() const override { return "SHOW [TEMPORARY] TABLES|DATABASES [[NOT] LIKE 'str'] [LIMIT expr]"; } + const char * getName() const override { return "SHOW [TEMPORARY] TABLES|DATABASES|CLUSTERS|CLUSTER 'name' [[NOT] LIKE 'str'] [LIMIT expr]"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; diff --git a/tests/queries/0_stateless/01293_show_create_cluster.sql b/tests/queries/0_stateless/01293_show_create_cluster.sql new file mode 100644 index 00000000000..af450680dac --- /dev/null +++ b/tests/queries/0_stateless/01293_show_create_cluster.sql @@ -0,0 +1,3 @@ +show clusters; +show clusters like 'test%' limit 1; +show cluster 'test_shard_localhost'; From 5bc7f67e616cdc81bc2827750d981995f6cdeb32 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 5 Jun 2020 14:54:54 +0300 Subject: [PATCH 023/329] Better metadata for select query and renames --- .../PushingToViewsBlockOutputStream.cpp | 2 +- src/Storages/IStorage.cpp | 41 ++++++-- src/Storages/IStorage.h | 38 ++++--- src/Storages/LiveView/StorageLiveView.h | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 23 ++--- src/Storages/MergeTree/MergeTreeData.h | 2 - src/Storages/StorageInMemoryMetadata.cpp | 34 ------- src/Storages/StorageInMemoryMetadata.h | 30 ------ src/Storages/StorageMaterializedView.cpp | 98 ++++--------------- src/Storages/StorageMaterializedView.h | 7 -- src/Storages/StorageView.cpp | 10 +- src/Storages/StorageView.h | 3 - src/Storages/TTLDescription.cpp | 2 +- src/Storages/TTLDescription.h | 3 +- src/Storages/ya.make | 2 + 15 files changed, 103 insertions(+), 193 deletions(-) diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 2c4792e184e..2c2e6972158 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -72,7 +72,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( StoragePtr inner_table = materialized_view->getTargetTable(); auto inner_table_id = inner_table->getStorageID(); - query = materialized_view->getInnerQuery(); + query = materialized_view->getSelectQuery().inner_query; std::unique_ptr insert = std::make_unique(); insert->table_id = inner_table_id; diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 496aa55d071..33daf0c298c 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -421,12 +421,12 @@ NamesAndTypesList IStorage::getVirtuals() const return {}; } -const StorageMetadataKeyField & IStorage::getPartitionKey() const +const KeyDescription & IStorage::getPartitionKey() const { return partition_key; } -void IStorage::setPartitionKey(const StorageMetadataKeyField & partition_key_) +void IStorage::setPartitionKey(const KeyDescription & partition_key_) { partition_key = partition_key_; } @@ -448,12 +448,12 @@ Names IStorage::getColumnsRequiredForPartitionKey() const return {}; } -const StorageMetadataKeyField & IStorage::getSortingKey() const +const KeyDescription & IStorage::getSortingKey() const { return sorting_key; } -void IStorage::setSortingKey(const StorageMetadataKeyField & sorting_key_) +void IStorage::setSortingKey(const KeyDescription & sorting_key_) { sorting_key = sorting_key_; } @@ -482,12 +482,12 @@ Names IStorage::getSortingKeyColumns() const return {}; } -const StorageMetadataKeyField & IStorage::getPrimaryKey() const +const KeyDescription & IStorage::getPrimaryKey() const { return primary_key; } -void IStorage::setPrimaryKey(const StorageMetadataKeyField & primary_key_) +void IStorage::setPrimaryKey(const KeyDescription & primary_key_) { primary_key = primary_key_; } @@ -516,12 +516,12 @@ Names IStorage::getPrimaryKeyColumns() const return {}; } -const StorageMetadataKeyField & IStorage::getSamplingKey() const +const KeyDescription & IStorage::getSamplingKey() const { return sampling_key; } -void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_) +void IStorage::setSamplingKey(const KeyDescription & sampling_key_) { sampling_key = sampling_key_; } @@ -654,4 +654,29 @@ ColumnDependencies IStorage::getColumnDependencies(const NameSet & updated_colum } +ASTPtr IStorage::getSettingsChanges() const +{ + return settings_changes->clone(); +} + +void IStorage::setSettingsChanges(const ASTPtr & settings_changes_) +{ + settings_changes = settings_changes_->clone(); +} + +const SelectQueryDescription & IStorage::getSelectQuery() const +{ + return select; +} + +void IStorage::setSelectQuery(const SelectQueryDescription & select_) +{ + select = select_; +} + +bool IStorage::hasSelectQuery() const +{ + return select.select_query != nullptr; +} + } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index a637c9c6881..cd7b2ad8a0e 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -15,7 +15,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -152,6 +154,15 @@ public: /// thread-unsafe part. lockStructure must be acquired const ConstraintsDescription & getConstraints() const; void setConstraints(ConstraintsDescription constraints_); + /// Storage settings + ASTPtr getSettingsChanges() const; + void setSettingsChanges(const ASTPtr & settings_changes_); + bool hasSettingsChanges() const { return settings_changes != nullptr; } + + const SelectQueryDescription & getSelectQuery() const; + void setSelectQuery(const SelectQueryDescription & select_); + bool hasSelectQuery() const; + /// Returns storage metadata copy. Direct modification of /// result structure doesn't affect storage. virtual StorageInMemoryMetadata getInMemoryMetadata() const; @@ -203,14 +214,17 @@ private: IndicesDescription secondary_indices; ConstraintsDescription constraints; - StorageMetadataKeyField partition_key; - StorageMetadataKeyField primary_key; - StorageMetadataKeyField sorting_key; - StorageMetadataKeyField sampling_key; + KeyDescription partition_key; + KeyDescription primary_key; + KeyDescription sorting_key; + KeyDescription sampling_key; TTLColumnsDescription column_ttls_by_name; TTLTableDescription table_ttl; + ASTPtr settings_changes; + SelectQueryDescription select; + private: RWLockImpl::LockHolder tryLockTimed( const RWLock & rwlock, RWLockImpl::Type type, const String & query_id, const SettingSeconds & acquire_timeout) const; @@ -443,10 +457,10 @@ public: virtual Strings getDataPaths() const { return {}; } /// Returns structure with partition key. - const StorageMetadataKeyField & getPartitionKey() const; + const KeyDescription & getPartitionKey() const; /// Set partition key for storage (methods bellow, are just wrappers for this /// struct). - void setPartitionKey(const StorageMetadataKeyField & partition_key_); + void setPartitionKey(const KeyDescription & partition_key_); /// Returns ASTExpressionList of partition key expression for storage or nullptr if there is none. ASTPtr getPartitionKeyAST() const { return partition_key.definition_ast; } /// Storage has user-defined (in CREATE query) partition key. @@ -458,10 +472,10 @@ public: /// Returns structure with sorting key. - const StorageMetadataKeyField & getSortingKey() const; + const KeyDescription & getSortingKey() const; /// Set sorting key for storage (methods bellow, are just wrappers for this /// struct). - void setSortingKey(const StorageMetadataKeyField & sorting_key_); + void setSortingKey(const KeyDescription & sorting_key_); /// Returns ASTExpressionList of sorting key expression for storage or nullptr if there is none. ASTPtr getSortingKeyAST() const { return sorting_key.definition_ast; } /// Storage has user-defined (in CREATE query) sorting key. @@ -475,10 +489,10 @@ public: Names getSortingKeyColumns() const; /// Returns structure with primary key. - const StorageMetadataKeyField & getPrimaryKey() const; + const KeyDescription & getPrimaryKey() const; /// Set primary key for storage (methods bellow, are just wrappers for this /// struct). - void setPrimaryKey(const StorageMetadataKeyField & primary_key_); + void setPrimaryKey(const KeyDescription & primary_key_); /// Returns ASTExpressionList of primary key expression for storage or nullptr if there is none. ASTPtr getPrimaryKeyAST() const { return primary_key.definition_ast; } /// Storage has user-defined (in CREATE query) sorting key. @@ -493,10 +507,10 @@ public: Names getPrimaryKeyColumns() const; /// Returns structure with sampling key. - const StorageMetadataKeyField & getSamplingKey() const; + const KeyDescription & getSamplingKey() const; /// Set sampling key for storage (methods bellow, are just wrappers for this /// struct). - void setSamplingKey(const StorageMetadataKeyField & sampling_key_); + void setSamplingKey(const KeyDescription & sampling_key_); /// Returns sampling expression AST for storage or nullptr if there is none. ASTPtr getSamplingKeyAST() const { return sampling_key.definition_ast; } /// Storage has user-defined (in CREATE query) sampling key. diff --git a/src/Storages/LiveView/StorageLiveView.h b/src/Storages/LiveView/StorageLiveView.h index fe62de224da..458e74eb506 100644 --- a/src/Storages/LiveView/StorageLiveView.h +++ b/src/Storages/LiveView/StorageLiveView.h @@ -165,6 +165,7 @@ public: const Context & context); private: + /// TODO move to common struct SelectQueryDescription StorageID select_table_id = StorageID::createEmpty(); /// Will be initialized in constructor ASTPtr inner_query; /// stored query : SELECT * FROM ( SELECT a FROM A) ASTPtr inner_subquery; /// stored query's innermost subquery if any diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 63d163a593e..8b046673556 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -131,7 +131,6 @@ MergeTreeData::MergeTreeData( : IStorage(table_id_) , global_context(context_) , merging_params(merging_params_) - , settings_ast(metadata.settings_ast) , require_part_metadata(require_part_metadata_) , relative_data_path(relative_data_path_) , broken_part_callback(broken_part_callback_) @@ -145,6 +144,7 @@ MergeTreeData::MergeTreeData( if (relative_data_path.empty()) throw Exception("MergeTree storages require data path", ErrorCodes::INCORRECT_FILE_NAME); + setSettingsChanges(metadata.settings_ast); const auto settings = getSettings(); setProperties(metadata, /*only_check*/ false, attach); @@ -153,7 +153,7 @@ MergeTreeData::MergeTreeData( if (metadata.sample_by_ast != nullptr) { - StorageMetadataKeyField candidate_sampling_key = StorageMetadataKeyField::getKeyFromAST(metadata.sample_by_ast, getColumns(), global_context); + KeyDescription candidate_sampling_key = KeyDescription::getKeyFromAST(metadata.sample_by_ast, getColumns(), global_context); const auto & pk_sample_block = getPrimaryKey().sample_block; if (!pk_sample_block.has(candidate_sampling_key.column_names[0]) && !attach @@ -265,8 +265,8 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const if (isSamplingKeyDefined()) metadata.sample_by_ast = getSamplingKeyAST()->clone(); - if (settings_ast) - metadata.settings_ast = settings_ast->clone(); + if (hasSettingsChanges()) + metadata.settings_ast = getSettingsChanges(); return metadata; } @@ -444,7 +444,7 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool { setColumns(std::move(metadata.columns)); - StorageMetadataKeyField new_sorting_key; + KeyDescription new_sorting_key; new_sorting_key.definition_ast = metadata.order_by_ast; new_sorting_key.column_names = std::move(new_sorting_key_columns); new_sorting_key.expression_list_ast = std::move(new_sorting_key_expr_list); @@ -453,7 +453,7 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool new_sorting_key.data_types = std::move(new_sorting_key_data_types); setSortingKey(new_sorting_key); - StorageMetadataKeyField new_primary_key; + KeyDescription new_primary_key; new_primary_key.definition_ast = metadata.primary_key_ast; new_primary_key.column_names = std::move(new_primary_key_columns); new_primary_key.expression_list_ast = std::move(new_primary_key_expr_list); @@ -472,7 +472,7 @@ namespace { ExpressionActionsPtr getCombinedIndicesExpression( - const StorageMetadataKeyField & key, + const KeyDescription & key, const IndicesDescription & indices, const ColumnsDescription & columns, const Context & context) @@ -523,7 +523,7 @@ ASTPtr MergeTreeData::extractKeyExpressionList(const ASTPtr & node) void MergeTreeData::initPartitionKey(ASTPtr partition_by_ast) { - StorageMetadataKeyField new_partition_key = StorageMetadataKeyField::getKeyFromAST(partition_by_ast, getColumns(), global_context); + KeyDescription new_partition_key = KeyDescription::getKeyFromAST(partition_by_ast, getColumns(), global_context); if (new_partition_key.expression_list_ast->children.empty()) return; @@ -1460,9 +1460,10 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S setTTLExpressions(metadata.columns, metadata.ttl_for_table_ast, /* only_check = */ true); - if (settings_ast) + if (hasSettingsChanges()) { - const auto & current_changes = settings_ast->as().changes; + + const auto & current_changes = getSettingsChanges()->as().changes; const auto & new_changes = metadata.settings_ast->as().changes; for (const auto & changed_setting : new_changes) { @@ -1601,7 +1602,7 @@ void MergeTreeData::changeSettings( MergeTreeSettings copy = *getSettings(); copy.applyChanges(new_changes); storage_settings.set(std::make_unique(copy)); - settings_ast = new_settings; + setSettingsChanges(new_settings); } } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 6df181e3f98..dcc6174ef5a 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -677,8 +677,6 @@ protected: friend struct ReplicatedMergeTreeTableMetadata; friend class StorageReplicatedMergeTree; - ASTPtr settings_ast; - bool require_part_metadata; String relative_data_path; diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index ee38637e118..1b7ec39b9e3 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -1,13 +1,5 @@ #include -#include -#include -#include -#include -#include -#include -#include - namespace DB { @@ -89,30 +81,4 @@ StorageInMemoryMetadata & StorageInMemoryMetadata::operator=(const StorageInMemo return *this; } -StorageMetadataKeyField StorageMetadataKeyField::getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context) -{ - StorageMetadataKeyField result; - result.definition_ast = definition_ast; - result.expression_list_ast = extractKeyExpressionList(definition_ast); - - if (result.expression_list_ast->children.empty()) - return result; - - const auto & children = result.expression_list_ast->children; - for (const auto & child : children) - result.column_names.emplace_back(child->getColumnName()); - - { - auto expr = result.expression_list_ast->clone(); - auto syntax_result = SyntaxAnalyzer(context).analyze(expr, columns.getAllPhysical()); - result.expression = ExpressionAnalyzer(expr, syntax_result, context).getActions(true); - result.sample_block = result.expression->getSampleBlock(); - } - - for (size_t i = 0; i < result.sample_block.columns(); ++i) - result.data_types.emplace_back(result.sample_block.getByPosition(i).type); - - return result; -} - } diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index 39bc8fd2b31..39374166d5e 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -3,7 +3,6 @@ #include #include #include -#include #include namespace DB @@ -44,33 +43,4 @@ struct StorageInMemoryMetadata StorageInMemoryMetadata & operator=(const StorageInMemoryMetadata & other); }; -/// Common structure for primary, partition and other storage keys -struct StorageMetadataKeyField -{ - /// User defined AST in CREATE/ALTER query. This field may be empty, but key - /// can exists because some of them maybe set implicitly (for example, - /// primary key in merge tree can be part of sorting key) - ASTPtr definition_ast; - - /// ASTExpressionList with key fields, example: (x, toStartOfMonth(date))). - ASTPtr expression_list_ast; - - /// Expression from expression_list_ast created by ExpressionAnalyzer. Useful, - /// when you need to get required columns for key, example: a, date, b. - ExpressionActionsPtr expression; - - /// Sample block with key columns (names, types, empty column) - Block sample_block; - - /// Column names in key definition, example: x, toStartOfMonth(date), a * b. - Names column_names; - - /// Types from sample block ordered in columns order. - DataTypes data_types; - - /// Parse key structure from key definition. Requires all columns, available - /// in storage. - static StorageMetadataKeyField getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context); -}; - } diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 1b8c6acb49f..313e75a169e 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -41,58 +42,6 @@ static inline String generateInnerTableName(const StorageID & view_id) return ".inner." + view_id.getTableName(); } -static StorageID extractDependentTableFromSelectQuery(ASTSelectQuery & query, const Context & context, bool add_default_db = true) -{ - if (add_default_db) - { - AddDefaultDatabaseVisitor visitor(context.getCurrentDatabase(), nullptr); - visitor.visit(query); - } - - if (auto db_and_table = getDatabaseAndTable(query, 0)) - { - return StorageID(db_and_table->database, db_and_table->table/*, db_and_table->uuid*/); - } - else if (auto subquery = extractTableExpression(query, 0)) - { - auto * ast_select = subquery->as(); - if (!ast_select) - throw Exception("Logical error while creating StorageMaterializedView. " - "Could not retrieve table name from select query.", - DB::ErrorCodes::LOGICAL_ERROR); - if (ast_select->list_of_selects->children.size() != 1) - throw Exception("UNION is not supported for MATERIALIZED VIEW", - ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); - - auto & inner_query = ast_select->list_of_selects->children.at(0); - - return extractDependentTableFromSelectQuery(inner_query->as(), context, false); - } - else - return StorageID::createEmpty(); -} - - -static void checkAllowedQueries(const ASTSelectQuery & query) -{ - if (query.prewhere() || query.final() || query.sampleSize()) - throw Exception("MATERIALIZED VIEW cannot have PREWHERE, SAMPLE or FINAL.", DB::ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); - - ASTPtr subquery = extractTableExpression(query, 0); - if (!subquery) - return; - - if (const auto * ast_select = subquery->as()) - { - if (ast_select->list_of_selects->children.size() != 1) - throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); - - const auto & inner_query = ast_select->list_of_selects->children.at(0); - - checkAllowedQueries(inner_query->as()); - } -} - StorageMaterializedView::StorageMaterializedView( const StorageID & table_id_, @@ -117,13 +66,8 @@ StorageMaterializedView::StorageMaterializedView( if (query.select->list_of_selects->children.size() != 1) throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); - select = query.select->clone(); - inner_query = query.select->list_of_selects->children.at(0); - - auto & select_query = inner_query->as(); - checkAllowedQueries(select_query); - - select_table_id = extractDependentTableFromSelectQuery(select_query, local_context); + auto select = SelectQueryDescription::getSelectQueryFromASTForMatView(query.select->clone(), local_context); + setSelectQuery(select); if (!has_inner_table) target_table_id = query.to_table_id; @@ -152,14 +96,14 @@ StorageMaterializedView::StorageMaterializedView( target_table_id = DatabaseCatalog::instance().getTable({manual_create_query->database, manual_create_query->table}, global_context)->getStorageID(); } - if (!select_table_id.empty()) - DatabaseCatalog::instance().addDependency(select_table_id, getStorageID()); + if (!select.select_table_id.empty()) + DatabaseCatalog::instance().addDependency(select.select_table_id, getStorageID()); } StorageInMemoryMetadata StorageMaterializedView::getInMemoryMetadata() const { StorageInMemoryMetadata result(getColumns(), getSecondaryIndices(), getConstraints()); - result.select = getSelectQuery(); + result.select = getSelectQuery().select_query; return result; } @@ -222,8 +166,9 @@ static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, void StorageMaterializedView::drop() { auto table_id = getStorageID(); - if (!select_table_id.empty()) - DatabaseCatalog::instance().removeDependency(select_table_id, table_id); + const auto & select_query = getSelectQuery(); + if (!select_query.select_table_id.empty()) + DatabaseCatalog::instance().removeDependency(select_query.select_table_id, table_id); if (has_inner_table && tryGetTargetTable()) executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_table_id); @@ -262,21 +207,12 @@ void StorageMaterializedView::alter( /// start modify query if (context.getSettingsRef().allow_experimental_alter_materialized_view_structure) { - auto & new_select = metadata.select->as(); + auto new_select = SelectQueryDescription::getSelectQueryFromASTForMatView(metadata.select, context); + const auto & old_select = getSelectQuery(); - if (new_select.list_of_selects->children.size() != 1) - throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); + DatabaseCatalog::instance().updateDependency(old_select.select_table_id, table_id, new_select.select_table_id, table_id); - auto & new_inner_query = new_select.list_of_selects->children.at(0); - auto & select_query = new_inner_query->as(); - checkAllowedQueries(select_query); - - auto new_select_table_id = extractDependentTableFromSelectQuery(select_query, context); - DatabaseCatalog::instance().updateDependency(select_table_id, table_id, new_select_table_id, table_id); - - select_table_id = new_select_table_id; - select = metadata.select; - inner_query = new_inner_query; + setSelectQuery(new_select); } /// end modify query @@ -349,15 +285,17 @@ void StorageMaterializedView::renameInMemory(const StorageID & new_table_id) } IStorage::renameInMemory(new_table_id); + const auto & select_query = getSelectQuery(); // TODO Actually we don't need to update dependency if MV has UUID, but then db and table name will be outdated - DatabaseCatalog::instance().updateDependency(select_table_id, old_table_id, select_table_id, getStorageID()); + DatabaseCatalog::instance().updateDependency(select_query.select_table_id, old_table_id, select_query.select_table_id, getStorageID()); } void StorageMaterializedView::shutdown() { + const auto & select_query = getSelectQuery(); /// Make sure the dependency is removed after DETACH TABLE - if (!select_table_id.empty()) - DatabaseCatalog::instance().removeDependency(select_table_id, getStorageID()); + if (!select_query.select_table_id.empty()) + DatabaseCatalog::instance().removeDependency(select_query.select_table_id, getStorageID()); } StoragePtr StorageMaterializedView::getTargetTable() const diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index fd8639abb6a..d5f81e2248e 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -19,8 +19,6 @@ public: std::string getName() const override { return "MaterializedView"; } bool isView() const override { return true; } - ASTPtr getSelectQuery() const { return select->clone(); } - ASTPtr getInnerQuery() const { return inner_query->clone(); } bool hasInnerTable() const { return has_inner_table; } StorageInMemoryMetadata getInMemoryMetadata() const override; @@ -76,14 +74,9 @@ public: Strings getDataPaths() const override; private: - /// Can be empty if SELECT query doesn't contain table - StorageID select_table_id = StorageID::createEmpty(); /// Will be initialized in constructor StorageID target_table_id = StorageID::createEmpty(); - ASTPtr select; - ASTPtr inner_query; - Context & global_context; bool has_inner_table = false; diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index c6b37a50aa9..2525ac48732 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -12,6 +12,7 @@ #include #include +#include #include @@ -45,7 +46,10 @@ StorageView::StorageView( if (!query.select) throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); - inner_query = query.select->ptr(); + SelectQueryDescription description; + + description.inner_query = query.select->ptr(); + setSelectQuery(description); } @@ -59,7 +63,7 @@ Pipes StorageView::read( { Pipes pipes; - ASTPtr current_inner_query = inner_query; + ASTPtr current_inner_query = getSelectQuery().inner_query; if (context.getSettings().enable_optimize_predicate_expression) current_inner_query = getRuntimeViewQuery(*query_info.query->as(), context); @@ -119,7 +123,7 @@ static void replaceTableNameWithSubquery(ASTSelectQuery * select_query, ASTPtr & ASTPtr StorageView::getRuntimeViewQuery(ASTSelectQuery * outer_query, const Context & context, bool normalize) { - auto runtime_view_query = inner_query->clone(); + auto runtime_view_query = getSelectQuery().inner_query->clone(); /// TODO: remove getTableExpressions and getTablesWithColumns { diff --git a/src/Storages/StorageView.h b/src/Storages/StorageView.h index 86550db83ce..179da5a07b9 100644 --- a/src/Storages/StorageView.h +++ b/src/Storages/StorageView.h @@ -33,9 +33,6 @@ public: ASTPtr getRuntimeViewQuery(ASTSelectQuery * outer_query, const Context & context, bool normalize); -private: - ASTPtr inner_query; - protected: StorageView( const StorageID & table_id_, diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index da9691aab4a..3bef8894971 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -59,7 +59,7 @@ TTLDescription TTLDescription::getTTLFromAST( const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, - const StorageMetadataKeyField & primary_key) + const KeyDescription & primary_key) { TTLDescription result; const auto * ttl_element = definition_ast->as(); diff --git a/src/Storages/TTLDescription.h b/src/Storages/TTLDescription.h index 99a145b8acc..1ad6960ee3b 100644 --- a/src/Storages/TTLDescription.h +++ b/src/Storages/TTLDescription.h @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -73,7 +74,7 @@ struct TTLDescription /// Parse TTL structure from definition. Able to parse both column and table /// TTLs. - static TTLDescription getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, const StorageMetadataKeyField & primary_key); + static TTLDescription getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, const KeyDescription & primary_key); }; /// Mapping from column name to column TTL diff --git a/src/Storages/ya.make b/src/Storages/ya.make index a28dd393929..b10a6194b7c 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -176,6 +176,8 @@ SRCS( VirtualColumnUtils.cpp extractKeyExpressionList.cpp TTLDescription.cpp + KeyDescription.cpp + SelectQueryDescription.cpp ) END() From 2b23d1aa3349b487d953f3d48e63fa552ba7f0c4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 5 Jun 2020 15:13:24 +0300 Subject: [PATCH 024/329] Fix reference --- src/Storages/IStorage.cpp | 9 ++++++--- src/Storages/IStorage.h | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 33daf0c298c..8f7c6869892 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -654,14 +654,17 @@ ColumnDependencies IStorage::getColumnDependencies(const NameSet & updated_colum } -ASTPtr IStorage::getSettingsChanges() const +const ASTPtr & IStorage::getSettingsChanges() const { - return settings_changes->clone(); + return settings_changes; } void IStorage::setSettingsChanges(const ASTPtr & settings_changes_) { - settings_changes = settings_changes_->clone(); + if (settings_changes_) + settings_changes = settings_changes_->clone(); + else + settings_changes = nullptr; } const SelectQueryDescription & IStorage::getSelectQuery() const diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index cd7b2ad8a0e..4f6787e0c61 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -155,7 +155,7 @@ public: /// thread-unsafe part. lockStructure must be acquired void setConstraints(ConstraintsDescription constraints_); /// Storage settings - ASTPtr getSettingsChanges() const; + const ASTPtr & getSettingsChanges() const; void setSettingsChanges(const ASTPtr & settings_changes_); bool hasSettingsChanges() const { return settings_changes != nullptr; } From abaf47f0cda4e9d6436e8aaece11e0e69e904ddc Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 5 Jun 2020 20:29:40 +0300 Subject: [PATCH 025/329] Make metadata single structure --- src/Databases/DatabaseOrdinary.cpp | 20 +- src/Storages/AlterCommands.cpp | 21 +- src/Storages/ConstraintsDescription.cpp | 15 ++ src/Storages/ConstraintsDescription.h | 3 + src/Storages/IStorage.cpp | 97 +++++---- src/Storages/IStorage.h | 34 +-- src/Storages/IndicesDescription.cpp | 37 ++++ src/Storages/IndicesDescription.h | 7 + src/Storages/KeyDescription.cpp | 69 +++++++ src/Storages/KeyDescription.h | 45 ++++ src/Storages/MergeTree/MergeTreeData.cpp | 193 +++++------------- src/Storages/MergeTree/MergeTreeData.h | 7 +- .../MergeTree/StorageFromMergeTreeDataPart.h | 5 - .../MergeTree/registerStorageMergeTree.cpp | 45 ++-- src/Storages/SelectQueryDescription.cpp | 117 +++++++++++ src/Storages/SelectQueryDescription.h | 27 +++ src/Storages/StorageInMemoryMetadata.cpp | 69 ------- src/Storages/StorageInMemoryMetadata.h | 35 ++-- src/Storages/StorageMaterializedView.cpp | 9 +- src/Storages/StorageMaterializedView.h | 2 - src/Storages/StorageMergeTree.cpp | 6 +- src/Storages/StorageReplicatedMergeTree.cpp | 29 +-- src/Storages/TTLDescription.cpp | 65 ++++++ src/Storages/TTLDescription.h | 8 +- 24 files changed, 576 insertions(+), 389 deletions(-) create mode 100644 src/Storages/KeyDescription.cpp create mode 100644 src/Storages/KeyDescription.h create mode 100644 src/Storages/SelectQueryDescription.cpp create mode 100644 src/Storages/SelectQueryDescription.h diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index b31752ad1b3..eec58ed9b33 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -253,9 +253,9 @@ void DatabaseOrdinary::alterTable( ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->indices, new_indices); ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->constraints, new_constraints); - if (metadata.select) + if (metadata.select.select_query) { - ast->replace(ast_create_query.select, metadata.select); + ast->replace(ast_create_query.select, metadata.select.select_query); } /// MaterializedView is one type of CREATE query without storage. @@ -263,17 +263,17 @@ void DatabaseOrdinary::alterTable( { ASTStorage & storage_ast = *ast_create_query.storage; /// ORDER BY may change, but cannot appear, it's required construction - if (metadata.order_by_ast && storage_ast.order_by) - storage_ast.set(storage_ast.order_by, metadata.order_by_ast); + if (metadata.sorting_key.definition_ast && storage_ast.order_by) + storage_ast.set(storage_ast.order_by, metadata.sorting_key.definition_ast); - if (metadata.primary_key_ast) - storage_ast.set(storage_ast.primary_key, metadata.primary_key_ast); + if (metadata.primary_key.definition_ast) + storage_ast.set(storage_ast.primary_key, metadata.primary_key.definition_ast); - if (metadata.ttl_for_table_ast) - storage_ast.set(storage_ast.ttl_table, metadata.ttl_for_table_ast); + if (metadata.table_ttl.definition_ast) + storage_ast.set(storage_ast.ttl_table, metadata.table_ttl.definition_ast); - if (metadata.settings_ast) - storage_ast.set(storage_ast.settings, metadata.settings_ast); + if (metadata.settings_changes) + storage_ast.set(storage_ast.settings, metadata.settings_changes); } statement = getObjectDefinitionFromCreateQuery(ast); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index ce70af2bb6a..13da6db8ed3 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -316,14 +316,14 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con } else if (type == MODIFY_ORDER_BY) { - if (!metadata.primary_key_ast && metadata.order_by_ast) + if (metadata.primary_key.definition_ast == nullptr && metadata.sorting_key.definition_ast != nullptr) { /// Primary and sorting key become independent after this ALTER so we have to /// save the old ORDER BY expression as the new primary key. - metadata.primary_key_ast = metadata.order_by_ast->clone(); + metadata.primary_key = metadata.sorting_key; } - metadata.order_by_ast = order_by; + metadata.sorting_key = KeyDescription::getKeyFromAST(order_by, metadata.columns, context); } else if (type == COMMENT_COLUMN) { @@ -430,15 +430,15 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con } else if (type == MODIFY_TTL) { - metadata.ttl_for_table_ast = ttl; + metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl, metadata.columns, context, metadata.primary_key); } else if (type == MODIFY_QUERY) { - metadata.select = select; + metadata.select = SelectQueryDescription::getSelectQueryFromASTForMatView(select, context); } else if (type == MODIFY_SETTING) { - auto & settings_from_storage = metadata.settings_ast->as().changes; + auto & settings_from_storage = metadata.settings_changes->as().changes; for (const auto & change : settings_changes) { auto finder = [&change](const SettingChange & c) { return c.name == change.name; }; @@ -465,8 +465,11 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con rename_visitor.visit(column_to_modify.ttl); }); } - if (metadata.ttl_for_table_ast) - rename_visitor.visit(metadata.ttl_for_table_ast); + if (metadata.table_ttl.definition_ast) + rename_visitor.visit(metadata.table_ttl.definition_ast); + + metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + metadata.table_ttl.definition_ast, metadata.columns, context, metadata.primary_key); for (auto & constraint : metadata.constraints.constraints) rename_visitor.visit(constraint); @@ -832,7 +835,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con } else if (command.type == AlterCommand::MODIFY_SETTING) { - if (metadata.settings_ast == nullptr) + if (metadata.settings_changes == nullptr) throw Exception{"Cannot alter settings, because table engine doesn't support settings changes", ErrorCodes::BAD_ARGUMENTS}; } else if (command.type == AlterCommand::RENAME_COLUMN) diff --git a/src/Storages/ConstraintsDescription.cpp b/src/Storages/ConstraintsDescription.cpp index d86796908a7..3d4f528302a 100644 --- a/src/Storages/ConstraintsDescription.cpp +++ b/src/Storages/ConstraintsDescription.cpp @@ -54,4 +54,19 @@ ConstraintsExpressions ConstraintsDescription::getExpressions(const DB::Context return res; } +ConstraintsDescription::ConstraintsDescription(const ConstraintsDescription & other) +{ + constraints.reserve(other.constraints.size()); + for (const auto & constraint : other.constraints) + constraints.emplace_back(constraint->clone()); +} + +ConstraintsDescription & ConstraintsDescription::operator=(const ConstraintsDescription & other) +{ + constraints.resize(other.constraints.size()); + for (size_t i = 0; i < constraints.size(); ++i) + constraints[i] = other.constraints[i]->clone(); + return *this; +} + } diff --git a/src/Storages/ConstraintsDescription.h b/src/Storages/ConstraintsDescription.h index f4da4376041..ddc9597f813 100644 --- a/src/Storages/ConstraintsDescription.h +++ b/src/Storages/ConstraintsDescription.h @@ -20,6 +20,9 @@ struct ConstraintsDescription static ConstraintsDescription parse(const String & str); ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const; + + ConstraintsDescription(const ConstraintsDescription & other); + ConstraintsDescription & operator=(const ConstraintsDescription & other); }; } diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 8f7c6869892..2ad89f559b5 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -34,23 +34,23 @@ namespace ErrorCodes const ColumnsDescription & IStorage::getColumns() const { - return columns; + return metadata.columns; } const IndicesDescription & IStorage::getSecondaryIndices() const { - return secondary_indices; + return metadata.secondary_indices; } bool IStorage::hasSecondaryIndices() const { - return !secondary_indices.empty(); + return !metadata.secondary_indices.empty(); } const ConstraintsDescription & IStorage::getConstraints() const { - return constraints; + return metadata.constraints; } Block IStorage::getSampleBlock() const @@ -292,17 +292,17 @@ void IStorage::setColumns(ColumnsDescription columns_) { if (columns_.getOrdinary().empty()) throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); - columns = std::move(columns_); + metadata.columns = std::move(columns_); } void IStorage::setSecondaryIndices(IndicesDescription secondary_indices_) { - secondary_indices = std::move(secondary_indices_); + metadata.secondary_indices = std::move(secondary_indices_); } void IStorage::setConstraints(ConstraintsDescription constraints_) { - constraints = std::move(constraints_); + metadata.constraints = std::move(constraints_); } bool IStorage::isVirtualColumn(const String & column_name) const @@ -373,11 +373,6 @@ TableStructureWriteLockHolder IStorage::lockExclusively(const String & query_id, return result; } -StorageInMemoryMetadata IStorage::getInMemoryMetadata() const -{ - return StorageInMemoryMetadata(getColumns(), getSecondaryIndices(), getConstraints()); -} - void IStorage::alter( const AlterCommands & params, const Context & context, @@ -385,8 +380,8 @@ void IStorage::alter( { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto table_id = getStorageID(); - StorageInMemoryMetadata metadata = getInMemoryMetadata(); - params.apply(metadata, context); + StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); + params.apply(old_metadata, context); DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata); setColumns(std::move(metadata.columns)); } @@ -423,135 +418,135 @@ NamesAndTypesList IStorage::getVirtuals() const const KeyDescription & IStorage::getPartitionKey() const { - return partition_key; + return metadata.partition_key; } void IStorage::setPartitionKey(const KeyDescription & partition_key_) { - partition_key = partition_key_; + metadata.partition_key = partition_key_; } bool IStorage::isPartitionKeyDefined() const { - return partition_key.definition_ast != nullptr; + return metadata.partition_key.definition_ast != nullptr; } bool IStorage::hasPartitionKey() const { - return !partition_key.column_names.empty(); + return !metadata.partition_key.column_names.empty(); } Names IStorage::getColumnsRequiredForPartitionKey() const { if (hasPartitionKey()) - return partition_key.expression->getRequiredColumns(); + return metadata.partition_key.expression->getRequiredColumns(); return {}; } const KeyDescription & IStorage::getSortingKey() const { - return sorting_key; + return metadata.sorting_key; } void IStorage::setSortingKey(const KeyDescription & sorting_key_) { - sorting_key = sorting_key_; + metadata.sorting_key = sorting_key_; } bool IStorage::isSortingKeyDefined() const { - return sorting_key.definition_ast != nullptr; + return metadata.sorting_key.definition_ast != nullptr; } bool IStorage::hasSortingKey() const { - return !sorting_key.column_names.empty(); + return !metadata.sorting_key.column_names.empty(); } Names IStorage::getColumnsRequiredForSortingKey() const { if (hasSortingKey()) - return sorting_key.expression->getRequiredColumns(); + return metadata.sorting_key.expression->getRequiredColumns(); return {}; } Names IStorage::getSortingKeyColumns() const { if (hasSortingKey()) - return sorting_key.column_names; + return metadata.sorting_key.column_names; return {}; } const KeyDescription & IStorage::getPrimaryKey() const { - return primary_key; + return metadata.primary_key; } void IStorage::setPrimaryKey(const KeyDescription & primary_key_) { - primary_key = primary_key_; + metadata.primary_key = primary_key_; } bool IStorage::isPrimaryKeyDefined() const { - return primary_key.definition_ast != nullptr; + return metadata.primary_key.definition_ast != nullptr; } bool IStorage::hasPrimaryKey() const { - return !primary_key.column_names.empty(); + return !metadata.primary_key.column_names.empty(); } Names IStorage::getColumnsRequiredForPrimaryKey() const { if (hasPrimaryKey()) - return primary_key.expression->getRequiredColumns(); + return metadata.primary_key.expression->getRequiredColumns(); return {}; } Names IStorage::getPrimaryKeyColumns() const { if (hasSortingKey()) - return primary_key.column_names; + return metadata.primary_key.column_names; return {}; } const KeyDescription & IStorage::getSamplingKey() const { - return sampling_key; + return metadata.sampling_key; } void IStorage::setSamplingKey(const KeyDescription & sampling_key_) { - sampling_key = sampling_key_; + metadata.sampling_key = sampling_key_; } bool IStorage::isSamplingKeyDefined() const { - return sampling_key.definition_ast != nullptr; + return metadata.sampling_key.definition_ast != nullptr; } bool IStorage::hasSamplingKey() const { - return !sampling_key.column_names.empty(); + return !metadata.sampling_key.column_names.empty(); } Names IStorage::getColumnsRequiredForSampling() const { if (hasSamplingKey()) - return sampling_key.expression->getRequiredColumns(); + return metadata.sampling_key.expression->getRequiredColumns(); return {}; } const TTLTableDescription & IStorage::getTableTTLs() const { - return table_ttl; + return metadata.table_ttl; } void IStorage::setTableTTLs(const TTLTableDescription & table_ttl_) { - table_ttl = table_ttl_; + metadata.table_ttl = table_ttl_; } bool IStorage::hasAnyTableTTL() const @@ -561,37 +556,37 @@ bool IStorage::hasAnyTableTTL() const const TTLColumnsDescription & IStorage::getColumnTTLs() const { - return column_ttls_by_name; + return metadata.column_ttls_by_name; } void IStorage::setColumnTTLs(const TTLColumnsDescription & column_ttls_by_name_) { - column_ttls_by_name = column_ttls_by_name_; + metadata.column_ttls_by_name = column_ttls_by_name_; } bool IStorage::hasAnyColumnTTL() const { - return !column_ttls_by_name.empty(); + return !metadata.column_ttls_by_name.empty(); } const TTLDescription & IStorage::getRowsTTL() const { - return table_ttl.rows_ttl; + return metadata.table_ttl.rows_ttl; } bool IStorage::hasRowsTTL() const { - return table_ttl.rows_ttl.expression != nullptr; + return metadata.table_ttl.rows_ttl.expression != nullptr; } const TTLDescriptions & IStorage::getMoveTTLs() const { - return table_ttl.move_ttl; + return metadata.table_ttl.move_ttl; } bool IStorage::hasAnyMoveTTL() const { - return !table_ttl.move_ttl.empty(); + return !metadata.table_ttl.move_ttl.empty(); } @@ -656,30 +651,30 @@ ColumnDependencies IStorage::getColumnDependencies(const NameSet & updated_colum const ASTPtr & IStorage::getSettingsChanges() const { - return settings_changes; + return metadata.settings_changes; } void IStorage::setSettingsChanges(const ASTPtr & settings_changes_) { if (settings_changes_) - settings_changes = settings_changes_->clone(); + metadata.settings_changes = settings_changes_->clone(); else - settings_changes = nullptr; + metadata.settings_changes = nullptr; } const SelectQueryDescription & IStorage::getSelectQuery() const { - return select; + return metadata.select; } void IStorage::setSelectQuery(const SelectQueryDescription & select_) { - select = select_; + metadata.select = select_; } bool IStorage::hasSelectQuery() const { - return select.select_query != nullptr; + return metadata.select.select_query != nullptr; } } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 4f6787e0c61..82012eacc03 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -10,12 +10,7 @@ #include #include #include -#include -#include -#include #include -#include -#include #include #include #include @@ -157,15 +152,13 @@ public: /// thread-unsafe part. lockStructure must be acquired /// Storage settings const ASTPtr & getSettingsChanges() const; void setSettingsChanges(const ASTPtr & settings_changes_); - bool hasSettingsChanges() const { return settings_changes != nullptr; } + bool hasSettingsChanges() const { return metadata.settings_changes != nullptr; } const SelectQueryDescription & getSelectQuery() const; void setSelectQuery(const SelectQueryDescription & select_); bool hasSelectQuery() const; - /// Returns storage metadata copy. Direct modification of - /// result structure doesn't affect storage. - virtual StorageInMemoryMetadata getInMemoryMetadata() const; + const StorageInMemoryMetadata & getInMemoryMetadata() const { return metadata; } Block getSampleBlock() const; /// ordinary + materialized. Block getSampleBlockWithVirtuals() const; /// ordinary + materialized + virtuals. @@ -210,21 +203,8 @@ private: StorageID storage_id; mutable std::mutex id_mutex; - ColumnsDescription columns; - IndicesDescription secondary_indices; - ConstraintsDescription constraints; - - KeyDescription partition_key; - KeyDescription primary_key; - KeyDescription sorting_key; - KeyDescription sampling_key; - - TTLColumnsDescription column_ttls_by_name; - TTLTableDescription table_ttl; - - ASTPtr settings_changes; - SelectQueryDescription select; + StorageInMemoryMetadata metadata; private: RWLockImpl::LockHolder tryLockTimed( const RWLock & rwlock, RWLockImpl::Type type, const String & query_id, const SettingSeconds & acquire_timeout) const; @@ -462,7 +442,7 @@ public: /// struct). void setPartitionKey(const KeyDescription & partition_key_); /// Returns ASTExpressionList of partition key expression for storage or nullptr if there is none. - ASTPtr getPartitionKeyAST() const { return partition_key.definition_ast; } + ASTPtr getPartitionKeyAST() const { return metadata.partition_key.definition_ast; } /// Storage has user-defined (in CREATE query) partition key. bool isPartitionKeyDefined() const; /// Storage has partition key. @@ -477,7 +457,7 @@ public: /// struct). void setSortingKey(const KeyDescription & sorting_key_); /// Returns ASTExpressionList of sorting key expression for storage or nullptr if there is none. - ASTPtr getSortingKeyAST() const { return sorting_key.definition_ast; } + ASTPtr getSortingKeyAST() const { return metadata.sorting_key.definition_ast; } /// Storage has user-defined (in CREATE query) sorting key. bool isSortingKeyDefined() const; /// Storage has sorting key. It means, that it contains at least one column. @@ -494,7 +474,7 @@ public: /// struct). void setPrimaryKey(const KeyDescription & primary_key_); /// Returns ASTExpressionList of primary key expression for storage or nullptr if there is none. - ASTPtr getPrimaryKeyAST() const { return primary_key.definition_ast; } + ASTPtr getPrimaryKeyAST() const { return metadata.primary_key.definition_ast; } /// Storage has user-defined (in CREATE query) sorting key. bool isPrimaryKeyDefined() const; /// Storage has primary key (maybe part of some other key). It means, that @@ -512,7 +492,7 @@ public: /// struct). void setSamplingKey(const KeyDescription & sampling_key_); /// Returns sampling expression AST for storage or nullptr if there is none. - ASTPtr getSamplingKeyAST() const { return sampling_key.definition_ast; } + ASTPtr getSamplingKeyAST() const { return metadata.sampling_key.definition_ast; } /// Storage has user-defined (in CREATE query) sampling key. bool isSamplingKeyDefined() const; /// Storage has sampling key. diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index f3afe8c03a2..6f8406d9a3f 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -19,6 +19,43 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; }; +IndexDescription::IndexDescription(const IndexDescription & other) + : definition_ast(other.definition_ast ? other.definition_ast->clone() : nullptr) + , expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr) + , name(other.name) + , type(other.type) + , expression(other.expression) /// actions never changed + , arguments(other.arguments) + , column_names(other.column_names) + , data_types(other.data_types) + , sample_block(other.sample_block) + , granularity(other.granularity) +{ +} + + +IndexDescription & IndexDescription::operator=(const IndexDescription & other) +{ + if (other.definition_ast) + definition_ast = other.definition_ast->clone(); + else + definition_ast.reset(); + + if (other.expression_list_ast) + expression_list_ast = other.expression_list_ast->clone(); + else + expression_list_ast.reset(); + + name = other.name; + type = other.type; + expression = other.expression; + arguments = other.arguments; + column_names = other.column_names; + data_types = other.data_types; + sample_block = other.sample_block; + granularity = other.granularity; + return *this; +} IndexDescription IndexDescription::getIndexFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context) { diff --git a/src/Storages/IndicesDescription.h b/src/Storages/IndicesDescription.h index 16932dda6a2..b4d225c6511 100644 --- a/src/Storages/IndicesDescription.h +++ b/src/Storages/IndicesDescription.h @@ -48,6 +48,13 @@ struct IndexDescription /// Parse index from definition AST static IndexDescription getIndexFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context); + + IndexDescription() = default; + + /// We need custom copy constructors because we don't want + /// unintentionaly share AST variables and modify them. + IndexDescription(const IndexDescription & other); + IndexDescription & operator=(const IndexDescription & other); }; /// All secondary indices in storage diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp new file mode 100644 index 00000000000..78c7990b614 --- /dev/null +++ b/src/Storages/KeyDescription.cpp @@ -0,0 +1,69 @@ +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ + +KeyDescription::KeyDescription(const KeyDescription & other) + : definition_ast(other.definition_ast ? other.definition_ast->clone() : nullptr) + , expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr) + , expression(other.expression) + , sample_block(other.sample_block) + , column_names(other.column_names) + , data_types(other.data_types) +{ +} + +KeyDescription & KeyDescription::operator=(const KeyDescription & other) +{ + if (other.definition_ast) + definition_ast = other.definition_ast->clone(); + else + definition_ast.reset(); + + if (other.expression_list_ast) + expression_list_ast = other.expression_list_ast->clone(); + expression = other.expression; + sample_block = other.sample_block; + column_names = other.column_names; + data_types = other.data_types; + return *this; +} + + +KeyDescription KeyDescription::getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, ASTPtr additional_key_expression) +{ + KeyDescription result; + result.definition_ast = definition_ast; + result.expression_list_ast = extractKeyExpressionList(definition_ast); + + if (additional_key_expression) + result.expression_list_ast->children.push_back(additional_key_expression); + + if (result.expression_list_ast->children.empty()) + return result; + + const auto & children = result.expression_list_ast->children; + for (const auto & child : children) + result.column_names.emplace_back(child->getColumnName()); + + { + auto expr = result.expression_list_ast->clone(); + auto syntax_result = SyntaxAnalyzer(context).analyze(expr, columns.getAllPhysical()); + result.expression = ExpressionAnalyzer(expr, syntax_result, context).getActions(true); + result.sample_block = result.expression->getSampleBlock(); + } + + for (size_t i = 0; i < result.sample_block.columns(); ++i) + result.data_types.emplace_back(result.sample_block.getByPosition(i).type); + + return result; +} + +} diff --git a/src/Storages/KeyDescription.h b/src/Storages/KeyDescription.h new file mode 100644 index 00000000000..980bd3baf00 --- /dev/null +++ b/src/Storages/KeyDescription.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ +/// Common structure for primary, partition and other storage keys +struct KeyDescription +{ + /// User defined AST in CREATE/ALTER query. This field may be empty, but key + /// can exists because some of them maybe set implicitly (for example, + /// primary key in merge tree can be part of sorting key) + ASTPtr definition_ast; + + /// ASTExpressionList with key fields, example: (x, toStartOfMonth(date))). + ASTPtr expression_list_ast; + + /// Expression from expression_list_ast created by ExpressionAnalyzer. Useful, + /// when you need to get required columns for key, example: a, date, b. + ExpressionActionsPtr expression; + + /// Sample block with key columns (names, types, empty column) + Block sample_block; + + /// Column names in key definition, example: x, toStartOfMonth(date), a * b. + Names column_names; + + /// Types from sample block ordered in columns order. + DataTypes data_types; + + /// Parse key structure from key definition. Requires all columns, available + /// in storage. + static KeyDescription getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, ASTPtr additional_key_expression = nullptr); + + KeyDescription() = default; + + /// We need custom copy constructors because we don't want + /// unintentionaly share AST variables and modify them. + KeyDescription(const KeyDescription & other); + KeyDescription & operator=(const KeyDescription & other); +}; + +} diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 8b046673556..a4d709a4681 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -144,23 +144,21 @@ MergeTreeData::MergeTreeData( if (relative_data_path.empty()) throw Exception("MergeTree storages require data path", ErrorCodes::INCORRECT_FILE_NAME); - setSettingsChanges(metadata.settings_ast); + setSettingsChanges(metadata.settings_changes); const auto settings = getSettings(); setProperties(metadata, /*only_check*/ false, attach); /// NOTE: using the same columns list as is read when performing actual merges. merging_params.check(getColumns().getAllPhysical()); - if (metadata.sample_by_ast != nullptr) + if (metadata.sampling_key.definition_ast != nullptr) { - KeyDescription candidate_sampling_key = KeyDescription::getKeyFromAST(metadata.sample_by_ast, getColumns(), global_context); - const auto & pk_sample_block = getPrimaryKey().sample_block; - if (!pk_sample_block.has(candidate_sampling_key.column_names[0]) && !attach + if (!pk_sample_block.has(metadata.sampling_key.column_names[0]) && !attach && !settings->compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility. throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS); - setSamplingKey(candidate_sampling_key); + setSamplingKey(metadata.sampling_key); } MergeTreeDataFormatVersion min_format_version(0); @@ -169,7 +167,8 @@ MergeTreeData::MergeTreeData( try { auto partition_by_ast = makeASTFunction("toYYYYMM", std::make_shared(date_column_name)); - initPartitionKey(partition_by_ast); + auto partition_key = KeyDescription::getKeyFromAST(partition_by_ast, getColumns(), global_context); + initPartitionKey(partition_key); if (minmax_idx_date_column_pos == -1) throw Exception("Could not find Date column", ErrorCodes::BAD_TYPE_OF_FIELD); @@ -184,11 +183,11 @@ MergeTreeData::MergeTreeData( else { is_custom_partitioned = true; - initPartitionKey(metadata.partition_by_ast); + initPartitionKey(metadata.partition_key); min_format_version = MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING; } - setTTLExpressions(metadata.columns, metadata.ttl_for_table_ast); + setTTLExpressions(metadata.columns, metadata.table_ttl); /// format_file always contained on any data path PathWithDisk version_file; @@ -245,32 +244,6 @@ MergeTreeData::MergeTreeData( LOG_WARNING(log, "{} Settings 'min_bytes_for_wide_part' and 'min_bytes_for_wide_part' will be ignored.", reason); } - -StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const -{ - StorageInMemoryMetadata metadata(getColumns(), getSecondaryIndices(), getConstraints()); - - if (isPartitionKeyDefined()) - metadata.partition_by_ast = getPartitionKeyAST()->clone(); - - if (isSortingKeyDefined()) - metadata.order_by_ast = getSortingKeyAST()->clone(); - - if (isPrimaryKeyDefined()) - metadata.primary_key_ast = getPrimaryKeyAST()->clone(); - - if (hasAnyTableTTL()) - metadata.ttl_for_table_ast = getTableTTLs().definition_ast->clone(); - - if (isSamplingKeyDefined()) - metadata.sample_by_ast = getSamplingKeyAST()->clone(); - - if (hasSettingsChanges()) - metadata.settings_ast = getSettingsChanges(); - - return metadata; -} - StoragePolicyPtr MergeTreeData::getStoragePolicy() const { return global_context.getStoragePolicy(getSettings()->storage_policy); @@ -306,35 +279,43 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool only_check, bool attach) { - if (!metadata.order_by_ast) + KeyDescription new_sorting_key = metadata.sorting_key; + KeyDescription new_primary_key = metadata.primary_key; + + if (!new_sorting_key.definition_ast) throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS); - ASTPtr new_sorting_key_expr_list = extractKeyExpressionList(metadata.order_by_ast); - ASTPtr new_primary_key_expr_list = metadata.primary_key_ast - ? extractKeyExpressionList(metadata.primary_key_ast) : new_sorting_key_expr_list->clone(); - if (merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) - new_sorting_key_expr_list->children.push_back(std::make_shared(merging_params.version_column)); + new_sorting_key = KeyDescription::getKeyFromAST( + metadata.sorting_key.definition_ast, + metadata.columns, + global_context, + std::make_shared(merging_params.version_column)); - size_t primary_key_size = new_primary_key_expr_list->children.size(); - size_t sorting_key_size = new_sorting_key_expr_list->children.size(); + /// Primary key not defined at all + if (new_primary_key.definition_ast == nullptr) + { + /// We copy sorting key, and restore definition_ast to empty value + new_primary_key = metadata.sorting_key; + new_primary_key.definition_ast = nullptr; + } + + size_t sorting_key_size = new_sorting_key.column_names.size(); + size_t primary_key_size = new_primary_key.column_names.size(); if (primary_key_size > sorting_key_size) throw Exception("Primary key must be a prefix of the sorting key, but its length: " + toString(primary_key_size) + " is greater than the sorting key length: " + toString(sorting_key_size), ErrorCodes::BAD_ARGUMENTS); - Names new_primary_key_columns; - Names new_sorting_key_columns; NameSet primary_key_columns_set; for (size_t i = 0; i < sorting_key_size; ++i) { - String sorting_key_column = new_sorting_key_expr_list->children[i]->getColumnName(); - new_sorting_key_columns.push_back(sorting_key_column); + const String & sorting_key_column = new_sorting_key.column_names[i]; if (i < primary_key_size) { - String pk_column = new_primary_key_expr_list->children[i]->getColumnName(); + const String & pk_column = new_primary_key.column_names[i]; if (pk_column != sorting_key_column) throw Exception("Primary key must be a prefix of the sorting key, but in position " + toString(i) + " its column is " + pk_column + ", not " + sorting_key_column, @@ -343,7 +324,6 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool if (!primary_key_columns_set.emplace(pk_column).second) throw Exception("Primary key contains duplicate columns", ErrorCodes::BAD_ARGUMENTS); - new_primary_key_columns.push_back(pk_column); } } @@ -355,6 +335,9 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool /// This is ALTER, not CREATE/ATTACH TABLE. Let us check that all new columns used in the sorting key /// expression have just been added (so that the sorting order is guaranteed to be valid with the new key). + Names new_primary_key_columns = new_primary_key.column_names; + Names new_sorting_key_columns = new_sorting_key.column_names; + ASTPtr added_key_column_expr_list = std::make_shared(); const auto & old_sorting_key_columns = getSortingKeyColumns(); for (size_t new_i = 0, old_i = 0; new_i < sorting_key_size; ++new_i) @@ -362,12 +345,12 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool if (old_i < old_sorting_key_columns.size()) { if (new_sorting_key_columns[new_i] != old_sorting_key_columns[old_i]) - added_key_column_expr_list->children.push_back(new_sorting_key_expr_list->children[new_i]); + added_key_column_expr_list->children.push_back(new_sorting_key.expression_list_ast->children[new_i]); else ++old_i; } else - added_key_column_expr_list->children.push_back(new_sorting_key_expr_list->children[new_i]); + added_key_column_expr_list->children.push_back(new_sorting_key.expression_list_ast->children[new_i]); } if (!added_key_column_expr_list->children.empty()) @@ -394,37 +377,9 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool } } - auto new_sorting_key_syntax = SyntaxAnalyzer(global_context).analyze(new_sorting_key_expr_list, all_columns); - auto new_sorting_key_expr = ExpressionAnalyzer(new_sorting_key_expr_list, new_sorting_key_syntax, global_context) - .getActions(false); - auto new_sorting_key_sample = - ExpressionAnalyzer(new_sorting_key_expr_list, new_sorting_key_syntax, global_context) - .getActions(true)->getSampleBlock(); - - checkKeyExpression(*new_sorting_key_expr, new_sorting_key_sample, "Sorting"); - - auto new_primary_key_syntax = SyntaxAnalyzer(global_context).analyze(new_primary_key_expr_list, all_columns); - auto new_primary_key_expr = ExpressionAnalyzer(new_primary_key_expr_list, new_primary_key_syntax, global_context) - .getActions(false); - - Block new_primary_key_sample; - DataTypes new_primary_key_data_types; - for (size_t i = 0; i < primary_key_size; ++i) - { - const auto & elem = new_sorting_key_sample.getByPosition(i); - new_primary_key_sample.insert(elem); - new_primary_key_data_types.push_back(elem.type); - } - - DataTypes new_sorting_key_data_types; - for (size_t i = 0; i < sorting_key_size; ++i) - { - new_sorting_key_data_types.push_back(new_sorting_key_sample.getByPosition(i).type); - } - if (!metadata.secondary_indices.empty()) { - std::set indices_names; + std::unordered_set indices_names; for (const auto & index : metadata.secondary_indices) { @@ -444,22 +399,8 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool { setColumns(std::move(metadata.columns)); - KeyDescription new_sorting_key; - new_sorting_key.definition_ast = metadata.order_by_ast; - new_sorting_key.column_names = std::move(new_sorting_key_columns); - new_sorting_key.expression_list_ast = std::move(new_sorting_key_expr_list); - new_sorting_key.expression = std::move(new_sorting_key_expr); - new_sorting_key.sample_block = std::move(new_sorting_key_sample); - new_sorting_key.data_types = std::move(new_sorting_key_data_types); setSortingKey(new_sorting_key); - KeyDescription new_primary_key; - new_primary_key.definition_ast = metadata.primary_key_ast; - new_primary_key.column_names = std::move(new_primary_key_columns); - new_primary_key.expression_list_ast = std::move(new_primary_key_expr_list); - new_primary_key.expression = std::move(new_primary_key_expr); - new_primary_key.sample_block = std::move(new_primary_key_sample); - new_primary_key.data_types = std::move(new_primary_key_data_types); setPrimaryKey(new_primary_key); setSecondaryIndices(metadata.secondary_indices); @@ -521,10 +462,8 @@ ASTPtr MergeTreeData::extractKeyExpressionList(const ASTPtr & node) } -void MergeTreeData::initPartitionKey(ASTPtr partition_by_ast) +void MergeTreeData::initPartitionKey(const KeyDescription & new_partition_key) { - KeyDescription new_partition_key = KeyDescription::getKeyFromAST(partition_by_ast, getColumns(), global_context); - if (new_partition_key.expression_list_ast->children.empty()) return; @@ -580,10 +519,10 @@ void MergeTreeData::initPartitionKey(ASTPtr partition_by_ast) } +/// Todo replace columns with TTL for columns void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns, - const ASTPtr & new_ttl_table_ast, bool only_check) + const TTLTableDescription & new_table_ttl, bool only_check) { - auto new_column_ttls_asts = new_columns.getColumnTTLs(); TTLColumnsDescription new_column_ttl_by_name = getColumnTTLs(); @@ -614,56 +553,24 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns, setColumnTTLs(new_column_ttl_by_name); } - if (new_ttl_table_ast) + if (new_table_ttl.definition_ast) { - TTLDescriptions update_move_ttl_entries; - TTLDescription update_rows_ttl_entry; - - bool seen_delete_ttl = false; - for (const auto & ttl_element_ptr : new_ttl_table_ast->children) + for (const auto & move_ttl : new_table_ttl.move_ttl) { - const auto * ttl_element = ttl_element_ptr->as(); - if (!ttl_element) - throw Exception("Unexpected AST element in TTL expression", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - if (ttl_element->destination_type == DataDestinationType::DELETE) + if (!getDestinationForTTL(move_ttl)) { - if (seen_delete_ttl) - { - throw Exception("More than one DELETE TTL expression is not allowed", ErrorCodes::BAD_TTL_EXPRESSION); - } - - update_rows_ttl_entry = TTLDescription::getTTLFromAST(ttl_element_ptr, new_columns, global_context, getPrimaryKey()); - - seen_delete_ttl = true; - } - else - { - auto new_ttl_entry = TTLDescription::getTTLFromAST(ttl_element_ptr, new_columns, global_context, getPrimaryKey()); - - if (!getDestinationForTTL(new_ttl_entry)) - { - String message; - if (new_ttl_entry.destination_type == DataDestinationType::DISK) - message = "No such disk " + backQuote(new_ttl_entry.destination_name) + " for given storage policy."; - else - message = "No such volume " + backQuote(new_ttl_entry.destination_name) + " for given storage policy."; - throw Exception(message, ErrorCodes::BAD_TTL_EXPRESSION); - } - - update_move_ttl_entries.emplace_back(std::move(new_ttl_entry)); + String message; + if (move_ttl.destination_type == DataDestinationType::DISK) + message = "No such disk " + backQuote(move_ttl.destination_name) + " for given storage policy."; + else + message = "No such volume " + backQuote(move_ttl.destination_name) + " for given storage policy."; + throw Exception(message, ErrorCodes::BAD_TTL_EXPRESSION); } } + if (!only_check) { - TTLTableDescription new_table_ttl - { - .definition_ast = new_ttl_table_ast, - .rows_ttl = update_rows_ttl_entry, - .move_ttl = update_move_ttl_entries, - }; - auto move_ttl_entries_lock = std::lock_guard(move_ttl_entries_mutex); setTableTTLs(new_table_ttl); } @@ -1458,13 +1365,13 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S setProperties(metadata, /* only_check = */ true); - setTTLExpressions(metadata.columns, metadata.ttl_for_table_ast, /* only_check = */ true); + setTTLExpressions(metadata.columns, metadata.table_ttl, /* only_check = */ true); if (hasSettingsChanges()) { const auto & current_changes = getSettingsChanges()->as().changes; - const auto & new_changes = metadata.settings_ast->as().changes; + const auto & new_changes = metadata.settings_changes->as().changes; for (const auto & changed_setting : new_changes) { if (MergeTreeSettings::findIndex(changed_setting.name) == MergeTreeSettings::npos) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index dcc6174ef5a..8b24bea7830 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -333,9 +333,6 @@ public: BrokenPartCallback broken_part_callback_ = [](const String &){}); - /// See comments about methods below in IStorage interface - StorageInMemoryMetadata getInMemoryMetadata() const override; - StoragePolicyPtr getStoragePolicy() const override; bool supportsPrewhere() const override { return true; } @@ -785,10 +782,10 @@ protected: void setProperties(const StorageInMemoryMetadata & metadata, bool only_check = false, bool attach = false); - void initPartitionKey(ASTPtr partition_by_ast); + void initPartitionKey(const KeyDescription & new_partition_key); void setTTLExpressions(const ColumnsDescription & columns, - const ASTPtr & new_ttl_table_ast, bool only_check = false); + const TTLTableDescription & new_table_ttl, bool only_check = false); void checkStoragePolicy(const StoragePolicyPtr & new_storage_policy) const; diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 4cecba1faff..3031402715a 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -39,11 +39,6 @@ public: return part->storage.mayBenefitFromIndexForIn(left_in_operand, query_context); } - StorageInMemoryMetadata getInMemoryMetadata() const override - { - return part->storage.getInMemoryMetadata(); - } - NamesAndTypesList getVirtuals() const override { return part->storage.getVirtuals(); diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 26a02f8904c..3d7c215b921 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -572,47 +572,46 @@ static StoragePtr create(const StorageFactory::Arguments & args) } String date_column_name; - ASTPtr partition_by_ast; - ASTPtr order_by_ast; - ASTPtr primary_key_ast; - ASTPtr sample_by_ast; - ASTPtr ttl_table_ast; - ASTPtr settings_ast; - IndicesDescription indices_description; - ConstraintsDescription constraints_description; + + StorageInMemoryMetadata metadata; + metadata.columns = args.columns; std::unique_ptr storage_settings = std::make_unique(args.context.getMergeTreeSettings()); if (is_extended_storage_def) { if (args.storage_def->partition_by) - partition_by_ast = args.storage_def->partition_by->ptr(); + metadata.partition_key = KeyDescription::getKeyFromAST( + args.storage_def->partition_by->ptr(), metadata.columns, args.context); if (!args.storage_def->order_by) throw Exception("You must provide an ORDER BY expression in the table definition. " "If you don't want this table to be sorted, use ORDER BY tuple()", ErrorCodes::BAD_ARGUMENTS); - order_by_ast = args.storage_def->order_by->ptr(); + metadata.sorting_key = KeyDescription::getKeyFromAST(args.storage_def->order_by->ptr(), metadata.columns, args.context); if (args.storage_def->primary_key) - primary_key_ast = args.storage_def->primary_key->ptr(); + metadata.primary_key = KeyDescription::getKeyFromAST(args.storage_def->primary_key->ptr(), metadata.columns, args.context); if (args.storage_def->sample_by) - sample_by_ast = args.storage_def->sample_by->ptr(); + metadata.sampling_key = KeyDescription::getKeyFromAST(args.storage_def->sample_by->ptr(), metadata.columns, args.context); if (args.storage_def->ttl_table) - ttl_table_ast = args.storage_def->ttl_table->ptr(); - + metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + args.storage_def->ttl_table->ptr(), + metadata.columns, + args.context, + metadata.primary_key); if (args.query.columns_list && args.query.columns_list->indices) for (auto & index : args.query.columns_list->indices->children) - indices_description.push_back(IndexDescription::getIndexFromAST(index, args.columns, args.context)); + metadata.secondary_indices.push_back(IndexDescription::getIndexFromAST(index, args.columns, args.context)); storage_settings->loadFromQuery(*args.storage_def); if (args.storage_def->settings) - settings_ast = args.storage_def->settings->ptr(); + metadata.settings_changes = args.storage_def->settings->ptr(); } else { @@ -627,12 +626,12 @@ static StoragePtr create(const StorageFactory::Arguments & args) /// If there is an expression for sampling if (arg_cnt - arg_num == 3) { - sample_by_ast = engine_args[arg_num]; + metadata.sampling_key = KeyDescription::getKeyFromAST(engine_args[arg_num], metadata.columns, args.context); ++arg_num; } /// Now only two parameters remain - primary_key, index_granularity. - order_by_ast = engine_args[arg_num]; + metadata.sorting_key = KeyDescription::getKeyFromAST(engine_args[arg_num], metadata.columns, args.context); ++arg_num; const auto * ast = engine_args[arg_num]->as(); @@ -648,18 +647,10 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (arg_num != arg_cnt) throw Exception("Wrong number of engine arguments.", ErrorCodes::BAD_ARGUMENTS); - if (!args.attach && !indices_description.empty() && !args.local_context.getSettingsRef().allow_experimental_data_skipping_indices) + if (!args.attach && !metadata.secondary_indices.empty() && !args.local_context.getSettingsRef().allow_experimental_data_skipping_indices) throw Exception("You must set the setting `allow_experimental_data_skipping_indices` to 1 " \ "before using data skipping indices.", ErrorCodes::BAD_ARGUMENTS); - StorageInMemoryMetadata metadata(args.columns, indices_description, args.constraints); - metadata.partition_by_ast = partition_by_ast; - metadata.order_by_ast = order_by_ast; - metadata.primary_key_ast = primary_key_ast; - metadata.ttl_for_table_ast = ttl_table_ast; - metadata.sample_by_ast = sample_by_ast; - metadata.settings_ast = settings_ast; - if (replicated) return StorageReplicatedMergeTree::create( zookeeper_path, replica_name, args.attach, args.table_id, args.relative_data_path, diff --git a/src/Storages/SelectQueryDescription.cpp b/src/Storages/SelectQueryDescription.cpp new file mode 100644 index 00000000000..3d2ba27a62c --- /dev/null +++ b/src/Storages/SelectQueryDescription.cpp @@ -0,0 +1,117 @@ +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ +extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW; +extern const int LOGICAL_ERROR; +} + +SelectQueryDescription::SelectQueryDescription(const SelectQueryDescription & other) + : select_table_id(other.select_table_id) + , select_query(other.select_query ? other.select_query->clone() : nullptr) + , inner_query(other.inner_query ? other.inner_query->clone() : nullptr) +{ +} + +SelectQueryDescription & SelectQueryDescription::SelectQueryDescription::operator=(const SelectQueryDescription & other) +{ + select_table_id = other.select_table_id; + if (other.select_query) + select_query = other.select_query->clone(); + else + select_query.reset(); + + if (other.inner_query) + inner_query = other.inner_query->clone(); + else + inner_query.reset(); + return *this; +} + + +namespace +{ + +StorageID extractDependentTableFromSelectQuery(ASTSelectQuery & query, const Context & context, bool add_default_db = true) +{ + if (add_default_db) + { + AddDefaultDatabaseVisitor visitor(context.getCurrentDatabase(), nullptr); + visitor.visit(query); + } + + if (auto db_and_table = getDatabaseAndTable(query, 0)) + { + return StorageID(db_and_table->database, db_and_table->table/*, db_and_table->uuid*/); + } + else if (auto subquery = extractTableExpression(query, 0)) + { + auto * ast_select = subquery->as(); + if (!ast_select) + throw Exception("Logical error while creating StorageMaterializedView. " + "Could not retrieve table name from select query.", + DB::ErrorCodes::LOGICAL_ERROR); + if (ast_select->list_of_selects->children.size() != 1) + throw Exception("UNION is not supported for MATERIALIZED VIEW", + ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); + + auto & inner_query = ast_select->list_of_selects->children.at(0); + + return extractDependentTableFromSelectQuery(inner_query->as(), context, false); + } + else + return StorageID::createEmpty(); +} + + +void checkAllowedQueries(const ASTSelectQuery & query) +{ + if (query.prewhere() || query.final() || query.sampleSize()) + throw Exception("MATERIALIZED VIEW cannot have PREWHERE, SAMPLE or FINAL.", DB::ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); + + ASTPtr subquery = extractTableExpression(query, 0); + if (!subquery) + return; + + if (const auto * ast_select = subquery->as()) + { + if (ast_select->list_of_selects->children.size() != 1) + throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); + + const auto & inner_query = ast_select->list_of_selects->children.at(0); + + checkAllowedQueries(inner_query->as()); + } +} + +} + +SelectQueryDescription SelectQueryDescription::getSelectQueryFromASTForMatView(const ASTPtr & select, const Context & context) +{ + auto & new_select = select->as(); + + if (new_select.list_of_selects->children.size() != 1) + throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); + + SelectQueryDescription result; + + result.inner_query = new_select.list_of_selects->children.at(0)->clone(); + + auto & select_query = result.inner_query->as(); + checkAllowedQueries(select_query); + result.select_table_id = extractDependentTableFromSelectQuery(select_query, context); + result.select_query = select->clone(); + + return result; +} + +} diff --git a/src/Storages/SelectQueryDescription.h b/src/Storages/SelectQueryDescription.h new file mode 100644 index 00000000000..ce3ca44c147 --- /dev/null +++ b/src/Storages/SelectQueryDescription.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +namespace DB +{ + +/// Select query for different view in storages +struct SelectQueryDescription +{ + /// Table id for select query + StorageID select_table_id = StorageID::createEmpty(); + /// Select query itself (ASTSelectWithUnionQuery) + ASTPtr select_query; + /// First query from select_query list + ASTPtr inner_query; + + /// Parse description from select query for materialized view. Also + /// validates query. + static SelectQueryDescription getSelectQueryFromASTForMatView(const ASTPtr & select, const Context & context); + + SelectQueryDescription() = default; + SelectQueryDescription(const SelectQueryDescription & other); + SelectQueryDescription & operator=(const SelectQueryDescription & other); +}; + +} diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 1b7ec39b9e3..67e0c7f6028 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -12,73 +12,4 @@ StorageInMemoryMetadata::StorageInMemoryMetadata( , constraints(constraints_) { } - -StorageInMemoryMetadata::StorageInMemoryMetadata(const StorageInMemoryMetadata & other) - : columns(other.columns) - , secondary_indices(other.secondary_indices) - , constraints(other.constraints) -{ - if (other.partition_by_ast) - partition_by_ast = other.partition_by_ast->clone(); - if (other.order_by_ast) - order_by_ast = other.order_by_ast->clone(); - if (other.primary_key_ast) - primary_key_ast = other.primary_key_ast->clone(); - if (other.ttl_for_table_ast) - ttl_for_table_ast = other.ttl_for_table_ast->clone(); - if (other.sample_by_ast) - sample_by_ast = other.sample_by_ast->clone(); - if (other.settings_ast) - settings_ast = other.settings_ast->clone(); - if (other.select) - select = other.select->clone(); -} - -StorageInMemoryMetadata & StorageInMemoryMetadata::operator=(const StorageInMemoryMetadata & other) -{ - if (this == &other) - return *this; - - columns = other.columns; - secondary_indices = other.secondary_indices; - constraints = other.constraints; - - if (other.partition_by_ast) - partition_by_ast = other.partition_by_ast->clone(); - else - partition_by_ast.reset(); - - if (other.order_by_ast) - order_by_ast = other.order_by_ast->clone(); - else - order_by_ast.reset(); - - if (other.primary_key_ast) - primary_key_ast = other.primary_key_ast->clone(); - else - primary_key_ast.reset(); - - if (other.ttl_for_table_ast) - ttl_for_table_ast = other.ttl_for_table_ast->clone(); - else - ttl_for_table_ast.reset(); - - if (other.sample_by_ast) - sample_by_ast = other.sample_by_ast->clone(); - else - sample_by_ast.reset(); - - if (other.settings_ast) - settings_ast = other.settings_ast->clone(); - else - settings_ast.reset(); - - if (other.select) - select = other.select->clone(); - else - select.reset(); - - return *this; -} - } diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index 39374166d5e..ba49cfa210d 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -1,9 +1,13 @@ #pragma once -#include -#include -#include #include +#include +#include +#include +#include +#include +#include + namespace DB { @@ -21,26 +25,25 @@ struct StorageInMemoryMetadata /// Table constraints. Currently supported for MergeTree only. ConstraintsDescription constraints; /// PARTITION BY expression. Currently supported for MergeTree only. - ASTPtr partition_by_ast = nullptr; + KeyDescription partition_key; + /// PRIMARY KEY expression. If absent, than equal to order_by_ast. + KeyDescription primary_key; /// ORDER BY expression. Required field for all MergeTree tables /// even in old syntax MergeTree(partition_key, order_by, ...) - ASTPtr order_by_ast = nullptr; - /// PRIMARY KEY expression. If absent, than equal to order_by_ast. - ASTPtr primary_key_ast = nullptr; - /// TTL expression for whole table. Supported for MergeTree only. - ASTPtr ttl_for_table_ast = nullptr; + KeyDescription sorting_key; /// SAMPLE BY expression. Supported for MergeTree only. - ASTPtr sample_by_ast = nullptr; + KeyDescription sampling_key; + /// Separate ttl expressions for columns + TTLColumnsDescription column_ttls_by_name; + /// TTL expressions for table (Move and Rows) + TTLTableDescription table_ttl; /// SETTINGS expression. Supported for MergeTree, Buffer and Kafka. - ASTPtr settings_ast = nullptr; - /// SELECT QUERY. Supported for MaterializedView only. - ASTPtr select = nullptr; + ASTPtr settings_changes; + /// SELECT QUERY. Supported for MaterializedView and View (have to support LiveView). + SelectQueryDescription select; - StorageInMemoryMetadata(const StorageInMemoryMetadata & other); StorageInMemoryMetadata() = default; StorageInMemoryMetadata(const ColumnsDescription & columns_, const IndicesDescription & secondary_indices_, const ConstraintsDescription & constraints_); - - StorageInMemoryMetadata & operator=(const StorageInMemoryMetadata & other); }; } diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 313e75a169e..6efba2b56a1 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -100,13 +100,6 @@ StorageMaterializedView::StorageMaterializedView( DatabaseCatalog::instance().addDependency(select.select_table_id, getStorageID()); } -StorageInMemoryMetadata StorageMaterializedView::getInMemoryMetadata() const -{ - StorageInMemoryMetadata result(getColumns(), getSecondaryIndices(), getConstraints()); - result.select = getSelectQuery().select_query; - return result; -} - QueryProcessingStage::Enum StorageMaterializedView::getQueryProcessingStage(const Context & context, QueryProcessingStage::Enum to_stage, const ASTPtr & query_ptr) const { return getTargetTable()->getQueryProcessingStage(context, to_stage, query_ptr); @@ -207,7 +200,7 @@ void StorageMaterializedView::alter( /// start modify query if (context.getSettingsRef().allow_experimental_alter_materialized_view_structure) { - auto new_select = SelectQueryDescription::getSelectQueryFromASTForMatView(metadata.select, context); + const auto & new_select = metadata.select; const auto & old_select = getSelectQuery(); DatabaseCatalog::instance().updateDependency(old_select.select_table_id, table_id, new_select.select_table_id, table_id); diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index d5f81e2248e..06bf659e05f 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -21,8 +21,6 @@ public: bool hasInnerTable() const { return has_inner_table; } - StorageInMemoryMetadata getInMemoryMetadata() const override; - bool supportsSampling() const override { return getTargetTable()->supportsSampling(); } bool supportsPrewhere() const override { return getTargetTable()->supportsPrewhere(); } bool supportsFinal() const override { return getTargetTable()->supportsFinal(); } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 4650485847c..65428a15876 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -260,7 +260,7 @@ void StorageMergeTree::alter( { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); - changeSettings(metadata.settings_ast, table_lock_holder); + changeSettings(metadata.settings_changes, table_lock_holder); DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata); } @@ -271,11 +271,11 @@ void StorageMergeTree::alter( auto merges_block = getActionLock(ActionLocks::PartsMerge); lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); - changeSettings(metadata.settings_ast, table_lock_holder); + changeSettings(metadata.settings_changes, table_lock_holder); /// Reinitialize primary key because primary key column types might have changed. setProperties(metadata); - setTTLExpressions(metadata.columns, metadata.ttl_for_table_ast); + setTTLExpressions(metadata.columns, metadata.table_ttl); DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index faa44ff7db1..f57dd0b5f32 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -483,20 +483,22 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column ParserNotEmptyExpressionList parser(false); auto new_sorting_key_expr_list = parseQuery(parser, metadata_diff.new_sorting_key, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + ASTPtr order_by_ast; if (new_sorting_key_expr_list->children.size() == 1) - metadata.order_by_ast = new_sorting_key_expr_list->children[0]; + order_by_ast = new_sorting_key_expr_list->children[0]; else { auto tuple = makeASTFunction("tuple"); tuple->arguments->children = new_sorting_key_expr_list->children; - metadata.order_by_ast = tuple; + order_by_ast = tuple; } + metadata.sorting_key = KeyDescription::getKeyFromAST(order_by_ast, metadata.columns, global_context); if (!isPrimaryKeyDefined()) { /// Primary and sorting key become independent after this ALTER so we have to /// save the old ORDER BY expression as the new primary key. - metadata.primary_key_ast = getSortingKeyAST()->clone(); + metadata.primary_key = getSortingKey(); } } @@ -509,7 +511,8 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column if (metadata_diff.ttl_table_changed) { ParserTTLExpressionList parser; - metadata.ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + auto ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl_for_table_ast, metadata.columns, global_context, metadata.primary_key); } } @@ -519,7 +522,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. setProperties(metadata); - setTTLExpressions(new_columns, metadata.ttl_for_table_ast); + setTTLExpressions(new_columns, metadata.table_ttl); } @@ -3295,7 +3298,7 @@ void StorageReplicatedMergeTree::alter( params.apply(metadata, query_context); - changeSettings(metadata.settings_ast, table_lock_holder); + changeSettings(metadata.settings_changes, table_lock_holder); DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id, metadata); return; @@ -3329,11 +3332,11 @@ void StorageReplicatedMergeTree::alter( params.apply(future_metadata, query_context); ReplicatedMergeTreeTableMetadata future_metadata_in_zk(*this); - if (ast_to_str(future_metadata.order_by_ast) != ast_to_str(current_metadata.order_by_ast)) - future_metadata_in_zk.sorting_key = serializeAST(*extractKeyExpressionList(future_metadata.order_by_ast)); + if (ast_to_str(future_metadata.sorting_key.definition_ast) != ast_to_str(current_metadata.sorting_key.definition_ast)) + future_metadata_in_zk.sorting_key = serializeAST(*future_metadata.sorting_key.expression_list_ast); - if (ast_to_str(future_metadata.ttl_for_table_ast) != ast_to_str(current_metadata.ttl_for_table_ast)) - future_metadata_in_zk.ttl_table = serializeAST(*future_metadata.ttl_for_table_ast); + if (ast_to_str(future_metadata.table_ttl.definition_ast) != ast_to_str(current_metadata.table_ttl.definition_ast)) + future_metadata_in_zk.ttl_table = serializeAST(*future_metadata.table_ttl.definition_ast); String new_indices_str = future_metadata.secondary_indices.toString(); if (new_indices_str != current_metadata.secondary_indices.toString()) @@ -3352,13 +3355,13 @@ void StorageReplicatedMergeTree::alter( ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, -1)); - if (ast_to_str(current_metadata.settings_ast) != ast_to_str(future_metadata.settings_ast)) + if (ast_to_str(current_metadata.settings_changes) != ast_to_str(future_metadata.settings_changes)) { lockStructureExclusively( table_lock_holder, query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout); /// Just change settings - current_metadata.settings_ast = future_metadata.settings_ast; - changeSettings(current_metadata.settings_ast, table_lock_holder); + current_metadata.settings_changes = future_metadata.settings_changes; + changeSettings(current_metadata.settings_changes, table_lock_holder); DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id, current_metadata); } diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 3bef8894971..ac77bd51b69 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -55,6 +55,41 @@ void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const Strin } +TTLDescription::TTLDescription(const TTLDescription & other) + : mode(other.mode) + , expression_ast(other.expression_ast ? other.expression_ast->clone() : nullptr) + , expression(other.expression) + , result_column(other.result_column) + , where_expression(other.where_expression) + , where_result_column(other.where_result_column) + , group_by_keys(other.group_by_keys) + , set_parts(other.set_parts) + , aggregate_descriptions(other.aggregate_descriptions) + , destination_type(other.destination_type) + , destination_name(other.destination_name) +{ +} + +TTLDescription & TTLDescription::operator=(const TTLDescription & other) +{ + mode = other.mode; + if (other.expression_ast) + expression_ast = other.expression_ast->clone(); + else + expression_ast.reset(); + + expression = other.expression; + result_column = other.result_column; + where_expression = other.where_expression; + where_result_column = other.where_result_column; + group_by_keys = other.group_by_keys; + set_parts = other.set_parts; + aggregate_descriptions = other.aggregate_descriptions; + destination_type = other.destination_type; + destination_name = other.destination_name; + return * this; +} + TTLDescription TTLDescription::getTTLFromAST( const ASTPtr & definition_ast, const ColumnsDescription & columns, @@ -195,4 +230,34 @@ TTLDescription TTLDescription::getTTLFromAST( return result; } + +TTLTableDescription TTLTableDescription::getTTLForTableFromAST( + const ASTPtr & definition_ast, + const ColumnsDescription & columns, + const Context & context, + const KeyDescription & primary_key) +{ + TTLTableDescription result; + if (!definition_ast) + return result; + + result.definition_ast = definition_ast->clone(); + + bool seen_delete_ttl = false; + for (const auto & ttl_element_ptr : definition_ast->children) + { + auto ttl = TTLDescription::getTTLFromAST(ttl_element_ptr, columns, context, primary_key); + if (ttl.destination_type == DataDestinationType::DELETE) + { + if (seen_delete_ttl) + throw Exception("More than one DELETE TTL expression is not allowed", ErrorCodes::BAD_TTL_EXPRESSION); + result.rows_ttl = ttl; + seen_delete_ttl = true; + } + else + result.move_ttl.emplace_back(std::move(ttl)); + } + return result; +} + } diff --git a/src/Storages/TTLDescription.h b/src/Storages/TTLDescription.h index 1ad6960ee3b..86e82e14c73 100644 --- a/src/Storages/TTLDescription.h +++ b/src/Storages/TTLDescription.h @@ -5,7 +5,6 @@ #include #include #include -#include #include namespace DB @@ -75,6 +74,10 @@ struct TTLDescription /// Parse TTL structure from definition. Able to parse both column and table /// TTLs. static TTLDescription getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, const KeyDescription & primary_key); + + TTLDescription() = default; + TTLDescription(const TTLDescription & other); + TTLDescription & operator=(const TTLDescription & other); }; /// Mapping from column name to column TTL @@ -94,6 +97,9 @@ struct TTLTableDescription /// Moving data TTL (to other disks or volumes) TTLDescriptions move_ttl; + + static TTLTableDescription getTTLForTableFromAST( + const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, const KeyDescription & primary_key); }; } From 75768413ad9882346524a674a4f2e5623746831a Mon Sep 17 00:00:00 2001 From: hexiaoting <“hewenting_ict@163.com”> Date: Mon, 8 Jun 2020 10:23:57 +0800 Subject: [PATCH 026/329] fix errors found by Style Check --- src/Interpreters/InterpreterShowTablesQuery.cpp | 15 +++++++-------- src/Parsers/ASTShowTablesQuery.cpp | 2 +- src/Parsers/ParserShowTablesQuery.cpp | 1 - .../0_stateless/01293_show_clusters.reference | 11 +++++++++++ ...create_cluster.sql => 01293_show_clusters.sql} | 0 5 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/01293_show_clusters.reference rename tests/queries/0_stateless/{01293_show_create_cluster.sql => 01293_show_clusters.sql} (100%) diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index a925466f72b..c647ea410c5 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -8,7 +8,6 @@ #include #include #include -#include namespace DB @@ -41,23 +40,23 @@ String InterpreterShowTablesQuery::getRewrittenQuery() rewritten_query << "SELECT cluster FROM system.clusters"; if (!query.like.empty()) - { - rewritten_query << " WHERE cluster " << (query.not_like ? "NOT " : "") << "LIKE " << std::quoted(query.like, '\''); - } + { + rewritten_query << " WHERE cluster " << (query.not_like ? "NOT " : "") << "LIKE " << std::quoted(query.like, '\''); + } - if (query.limit_length) + if (query.limit_length) rewritten_query << " LIMIT " << query.limit_length; - return rewritten_query.str(); + return rewritten_query.str(); } else if (query.cluster) { std::stringstream rewritten_query; - rewritten_query << "SELECT * FROM system.clusters"; + rewritten_query << "SELECT * FROM system.clusters"; rewritten_query << " WHERE cluster = " << std::quoted(query.cluster_str, '\''); - return rewritten_query.str(); + return rewritten_query.str(); } if (query.temporary && !query.from.empty()) diff --git a/src/Parsers/ASTShowTablesQuery.cpp b/src/Parsers/ASTShowTablesQuery.cpp index 39904061200..25a638c77d4 100644 --- a/src/Parsers/ASTShowTablesQuery.cpp +++ b/src/Parsers/ASTShowTablesQuery.cpp @@ -35,7 +35,7 @@ void ASTShowTablesQuery::formatQueryImpl(const FormatSettings & settings, Format else if (cluster) { settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CLUSTER" << (settings.hilite ? hilite_none : ""); - settings.ostr << " " << backQuoteIfNeed(cluster_str); + settings.ostr << " " << backQuoteIfNeed(cluster_str); } else { diff --git a/src/Parsers/ParserShowTablesQuery.cpp b/src/Parsers/ParserShowTablesQuery.cpp index fb29b6d99cd..2a0ec84461b 100644 --- a/src/Parsers/ParserShowTablesQuery.cpp +++ b/src/Parsers/ParserShowTablesQuery.cpp @@ -9,7 +9,6 @@ #include #include -#include namespace DB diff --git a/tests/queries/0_stateless/01293_show_clusters.reference b/tests/queries/0_stateless/01293_show_clusters.reference new file mode 100644 index 00000000000..b25a9a4d174 --- /dev/null +++ b/tests/queries/0_stateless/01293_show_clusters.reference @@ -0,0 +1,11 @@ +test_cluster_two_shards +test_cluster_two_shards +test_cluster_two_shards_localhost +test_cluster_two_shards_localhost +test_shard_localhost +test_shard_localhost[1] +test_shard_localhost_secure +test_unavailable_shard +test_unavailable_shard +test_cluster_two_shards +test_shard_localhost 1 1 1 localhost ::1 9000 1 default 0 0 diff --git a/tests/queries/0_stateless/01293_show_create_cluster.sql b/tests/queries/0_stateless/01293_show_clusters.sql similarity index 100% rename from tests/queries/0_stateless/01293_show_create_cluster.sql rename to tests/queries/0_stateless/01293_show_clusters.sql From 71e43934b7c41ce4e5472d39f9796793e6c1e5a3 Mon Sep 17 00:00:00 2001 From: hexiaoting <“hewenting_ict@163.com”> Date: Mon, 8 Jun 2020 10:29:53 +0800 Subject: [PATCH 027/329] Fix rest errors found by style check --- src/Interpreters/InterpreterShowTablesQuery.cpp | 2 +- src/Parsers/ParserShowTablesQuery.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index c647ea410c5..10447e52464 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -37,7 +37,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() if (query.clusters) { std::stringstream rewritten_query; - rewritten_query << "SELECT cluster FROM system.clusters"; + rewritten_query << "SELECT cluster FROM system.clusters"; if (!query.like.empty()) { diff --git a/src/Parsers/ParserShowTablesQuery.cpp b/src/Parsers/ParserShowTablesQuery.cpp index 2a0ec84461b..c60e442542d 100644 --- a/src/Parsers/ParserShowTablesQuery.cpp +++ b/src/Parsers/ParserShowTablesQuery.cpp @@ -70,11 +70,11 @@ bool ParserShowTablesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec { query->cluster = true; - String cluster_str; - if (!parseIdentifierOrStringLiteral(pos, expected, cluster_str)) - return false; - - query->cluster_str = std::move(cluster_str); + String cluster_str; + if (!parseIdentifierOrStringLiteral(pos, expected, cluster_str)) + return false; + + query->cluster_str = std::move(cluster_str); } else { From 2226f79f1662e79a1a4b4a71ffaf93de9f835aea Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 8 Jun 2020 15:57:25 +0300 Subject: [PATCH 028/329] Fix some bugs --- src/Storages/MergeTree/KeyCondition.cpp | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 13 +++++++++++-- src/Storages/MergeTree/registerStorageMergeTree.cpp | 6 ++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index dad73b6a003..9038fd684f1 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -524,6 +524,7 @@ bool KeyCondition::canConstantBeWrappedByMonotonicFunctions( Field & out_value, DataTypePtr & out_type) { + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "KEY EXPR EXMPTY: {}", (key_expr == nullptr)); String expr_name = node->getColumnName(); const auto & sample_block = key_expr->getSampleBlock(); if (!sample_block.has(expr_name)) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index d6437764653..b855faecf22 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -278,25 +278,31 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool only_check, bool attach) { - KeyDescription new_sorting_key = metadata.sorting_key; KeyDescription new_primary_key = metadata.primary_key; - if (!new_sorting_key.definition_ast) + if (!metadata.sorting_key.definition_ast) throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS); + KeyDescription new_sorting_key; if (merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) new_sorting_key = KeyDescription::getKeyFromAST( metadata.sorting_key.definition_ast, metadata.columns, global_context, std::make_shared(merging_params.version_column)); + else + new_sorting_key = metadata.sorting_key; /// Primary key not defined at all if (new_primary_key.definition_ast == nullptr) { + LOG_DEBUG(log, "PRIMARY KEY EMPTY, MAKING COPY"); /// We copy sorting key, and restore definition_ast to empty value new_primary_key = metadata.sorting_key; new_primary_key.definition_ast = nullptr; + LOG_DEBUG(log, "NEW PK DEF NULLPTR: {}", new_primary_key.definition_ast == nullptr); + LOG_DEBUG(log, "NEW PK EXPR NULLPTR: {}", new_primary_key.expression == nullptr); + LOG_DEBUG(log, "NEW PK COLUMN NAMES SIZE: {}", new_primary_key.column_names.size()); } size_t sorting_key_size = new_sorting_key.column_names.size(); @@ -402,9 +408,12 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool setPrimaryKey(new_primary_key); + setSecondaryIndices(metadata.secondary_indices); setConstraints(metadata.constraints); + LOG_DEBUG(log, "HAS PRIMARY KEY {}", hasPrimaryKey()); + LOG_DEBUG(log, "IS PRIMARY KEY {}", isPrimaryKeyDefined()); } } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index a012f9c00a6..d61418edfe7 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -491,9 +491,11 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (is_extended_storage_def) { + ASTPtr partition_by_key; if (args.storage_def->partition_by) - metadata.partition_key = KeyDescription::getKeyFromAST( - args.storage_def->partition_by->ptr(), metadata.columns, args.context); + partition_by_key = args.storage_def->partition_by->ptr(); + + metadata.partition_key = KeyDescription::getKeyFromAST(partition_by_key, metadata.columns, args.context); if (!args.storage_def->order_by) throw Exception("You must provide an ORDER BY expression in the table definition. " From 638cd50ce4e31cffd4cc70f3c417d4f88f35ab2a Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 8 Jun 2020 17:18:38 +0300 Subject: [PATCH 029/329] Fix several bugs --- src/Storages/KeyDescription.cpp | 9 +++--- src/Storages/MergeTree/MergeTreeData.cpp | 6 ++-- src/Storages/StorageInMemoryMetadata.cpp | 35 ++++++++++++++++++++++++ src/Storages/StorageInMemoryMetadata.h | 3 ++ 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index 78c7990b614..e59693f5343 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -46,9 +46,6 @@ KeyDescription KeyDescription::getKeyFromAST(const ASTPtr & definition_ast, cons if (additional_key_expression) result.expression_list_ast->children.push_back(additional_key_expression); - if (result.expression_list_ast->children.empty()) - return result; - const auto & children = result.expression_list_ast->children; for (const auto & child : children) result.column_names.emplace_back(child->getColumnName()); @@ -56,8 +53,10 @@ KeyDescription KeyDescription::getKeyFromAST(const ASTPtr & definition_ast, cons { auto expr = result.expression_list_ast->clone(); auto syntax_result = SyntaxAnalyzer(context).analyze(expr, columns.getAllPhysical()); - result.expression = ExpressionAnalyzer(expr, syntax_result, context).getActions(true); - result.sample_block = result.expression->getSampleBlock(); + /// In expression we also need to store source columns + result.expression = ExpressionAnalyzer(expr, syntax_result, context).getActions(false); + /// In sample block we use just key columns + result.sample_block = ExpressionAnalyzer(expr, syntax_result, context).getActions(true)->getSampleBlock(); } for (size_t i = 0; i < result.sample_block.columns(); ++i) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index b855faecf22..79382aaa58d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -296,13 +296,9 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool /// Primary key not defined at all if (new_primary_key.definition_ast == nullptr) { - LOG_DEBUG(log, "PRIMARY KEY EMPTY, MAKING COPY"); /// We copy sorting key, and restore definition_ast to empty value new_primary_key = metadata.sorting_key; new_primary_key.definition_ast = nullptr; - LOG_DEBUG(log, "NEW PK DEF NULLPTR: {}", new_primary_key.definition_ast == nullptr); - LOG_DEBUG(log, "NEW PK EXPR NULLPTR: {}", new_primary_key.expression == nullptr); - LOG_DEBUG(log, "NEW PK COLUMN NAMES SIZE: {}", new_primary_key.column_names.size()); } size_t sorting_key_size = new_sorting_key.column_names.size(); @@ -400,6 +396,8 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool } } + checkKeyExpression(*new_sorting_key.expression, new_sorting_key.sample_block, "Sorting"); + if (!only_check) { setColumns(std::move(metadata.columns)); diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 67e0c7f6028..0f8b88e691b 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -12,4 +12,39 @@ StorageInMemoryMetadata::StorageInMemoryMetadata( , constraints(constraints_) { } + +StorageInMemoryMetadata::StorageInMemoryMetadata(const StorageInMemoryMetadata & other) + : columns(other.columns) + , secondary_indices(other.secondary_indices) + , constraints(other.constraints) + , partition_key(other.partition_key) + , primary_key(other.primary_key) + , sorting_key(other.sorting_key) + , sampling_key(other.sampling_key) + , column_ttls_by_name(other.column_ttls_by_name) + , table_ttl(other.table_ttl) + , settings_changes(other.settings_changes ? other.settings_changes->clone() : nullptr) + , select(other.select) +{ +} + +StorageInMemoryMetadata & StorageInMemoryMetadata::operator=(const StorageInMemoryMetadata & other) +{ + columns = other.columns; + secondary_indices = other.secondary_indices; + constraints = other.constraints; + partition_key = other.partition_key; + primary_key = other.primary_key; + sorting_key = other.sorting_key; + sampling_key = other.sampling_key; + column_ttls_by_name = other.column_ttls_by_name; + table_ttl = other.table_ttl; + if (other.settings_changes) + settings_changes = other.settings_changes->clone(); + else + settings_changes.reset(); + select = other.select; + return *this; +} + } diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index ba49cfa210d..889f8e49f69 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -44,6 +44,9 @@ struct StorageInMemoryMetadata StorageInMemoryMetadata() = default; StorageInMemoryMetadata(const ColumnsDescription & columns_, const IndicesDescription & secondary_indices_, const ConstraintsDescription & constraints_); + + StorageInMemoryMetadata(const StorageInMemoryMetadata & other); + StorageInMemoryMetadata & operator=(const StorageInMemoryMetadata & other); }; } From 23549399946061ab2482a3989f84eeb43e843788 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 8 Jun 2020 17:19:20 +0300 Subject: [PATCH 030/329] Remove log debug --- src/Storages/MergeTree/KeyCondition.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 9038fd684f1..dad73b6a003 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -524,7 +524,6 @@ bool KeyCondition::canConstantBeWrappedByMonotonicFunctions( Field & out_value, DataTypePtr & out_type) { - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "KEY EXPR EXMPTY: {}", (key_expr == nullptr)); String expr_name = node->getColumnName(); const auto & sample_block = key_expr->getSampleBlock(); if (!sample_block.has(expr_name)) From a30b72a1284331eafdec59db56c16695761839ce Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 8 Jun 2020 17:21:12 +0300 Subject: [PATCH 031/329] Less debug --- src/Storages/MergeTree/MergeTreeData.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 79382aaa58d..0d8f02ff0b4 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -410,8 +410,6 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool setSecondaryIndices(metadata.secondary_indices); setConstraints(metadata.constraints); - LOG_DEBUG(log, "HAS PRIMARY KEY {}", hasPrimaryKey()); - LOG_DEBUG(log, "IS PRIMARY KEY {}", isPrimaryKeyDefined()); } } From 37a1d128c9d06594a5f4dd3e3731cdfe4d38a369 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 8 Jun 2020 17:22:11 +0300 Subject: [PATCH 032/329] Better --- src/Storages/MergeTree/MergeTreeData.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0d8f02ff0b4..587e3e5622e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -400,16 +400,12 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool if (!only_check) { + /// Other parts of metadata initialized is separate methods setColumns(std::move(metadata.columns)); - - setSortingKey(new_sorting_key); - - setPrimaryKey(new_primary_key); - - setSecondaryIndices(metadata.secondary_indices); - setConstraints(metadata.constraints); + setSortingKey(new_sorting_key); + setPrimaryKey(new_primary_key); } } From d625162fce75fbbcde852a55325e89169de128da Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 8 Jun 2020 17:23:32 +0300 Subject: [PATCH 033/329] Better comment --- src/Storages/MergeTree/MergeTreeData.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 587e3e5622e..f2196b91446 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -296,7 +296,10 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool /// Primary key not defined at all if (new_primary_key.definition_ast == nullptr) { - /// We copy sorting key, and restore definition_ast to empty value + /// We copy sorting key, and restore definition_ast to empty value, + /// because in merge tree code we sometimes chech, that our primary key + /// is fake (copied from sorting key, i.e. isPrimaryKeyDefined() == + /// false, but hasSortingKey() == true) new_primary_key = metadata.sorting_key; new_primary_key.definition_ast = nullptr; } From 7b63a88120bb6576b5787caa9609e603655f8ace Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 8 Jun 2020 18:19:41 +0300 Subject: [PATCH 034/329] Remove redundant error code --- src/Storages/MergeTree/MergeTreeData.cpp | 1 - src/Storages/StorageMaterializedView.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index f2196b91446..593498d5e84 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -106,7 +106,6 @@ namespace ErrorCodes extern const int READONLY_SETTING; extern const int ABORTED; extern const int UNKNOWN_PART_TYPE; - extern const int UNEXPECTED_AST_STRUCTURE; extern const int UNKNOWN_DISK; extern const int NOT_ENOUGH_SPACE; extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 74718f02d47..a7a59f0e9b9 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -30,7 +30,6 @@ namespace DB namespace ErrorCodes { extern const int NOT_IMPLEMENTED; - extern const int LOGICAL_ERROR; extern const int INCORRECT_QUERY; extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW; } From a5a9048109e15cb3daef1849141d5c9919c8decb Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 8 Jun 2020 21:23:26 +0300 Subject: [PATCH 035/329] At least fixed gcc-9 build --- src/Storages/StorageReplicatedMergeTree.cpp | 34 ++++++++++----------- src/Storages/StorageReplicatedMergeTree.h | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f57dd0b5f32..dc61ab110cd 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -160,7 +160,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( bool attach, const StorageID & table_id_, const String & relative_data_path_, - const StorageInMemoryMetadata & metadata, + const StorageInMemoryMetadata & metadata_, Context & context_, const String & date_column_name, const MergingParams & merging_params_, @@ -168,7 +168,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( bool has_force_restore_data_flag) : MergeTreeData(table_id_, relative_data_path_, - metadata, + metadata_, context_, date_column_name, merging_params_, @@ -472,9 +472,9 @@ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff) { - StorageInMemoryMetadata metadata = getInMemoryMetadata(); - if (new_columns != metadata.columns) - metadata.columns = new_columns; + StorageInMemoryMetadata current_metadata = getInMemoryMetadata(); + if (new_columns != current_metadata.columns) + current_metadata.columns = new_columns; if (!metadata_diff.empty()) { @@ -492,37 +492,37 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column tuple->arguments->children = new_sorting_key_expr_list->children; order_by_ast = tuple; } - metadata.sorting_key = KeyDescription::getKeyFromAST(order_by_ast, metadata.columns, global_context); + current_metadata.sorting_key = KeyDescription::getKeyFromAST(order_by_ast, current_metadata.columns, global_context); if (!isPrimaryKeyDefined()) { /// Primary and sorting key become independent after this ALTER so we have to /// save the old ORDER BY expression as the new primary key. - metadata.primary_key = getSortingKey(); + current_metadata.primary_key = getSortingKey(); } } if (metadata_diff.skip_indices_changed) - metadata.secondary_indices = IndicesDescription::parse(metadata_diff.new_skip_indices, new_columns, global_context); + current_metadata.secondary_indices = IndicesDescription::parse(metadata_diff.new_skip_indices, new_columns, global_context); if (metadata_diff.constraints_changed) - metadata.constraints = ConstraintsDescription::parse(metadata_diff.new_constraints); + current_metadata.constraints = ConstraintsDescription::parse(metadata_diff.new_constraints); if (metadata_diff.ttl_table_changed) { ParserTTLExpressionList parser; auto ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl_for_table_ast, metadata.columns, global_context, metadata.primary_key); + current_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl_for_table_ast, current_metadata.columns, global_context, current_metadata.primary_key); } } auto table_id = getStorageID(); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(global_context, table_id, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(global_context, table_id, current_metadata); /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. - setProperties(metadata); - setTTLExpressions(new_columns, metadata.table_ttl); + setProperties(current_metadata); + setTTLExpressions(new_columns, current_metadata.table_ttl); } @@ -3294,13 +3294,13 @@ void StorageReplicatedMergeTree::alter( table_lock_holder, query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout); /// We don't replicate storage_settings_ptr ALTER. It's local operation. /// Also we don't upgrade alter lock to table structure lock. - StorageInMemoryMetadata metadata = getInMemoryMetadata(); - params.apply(metadata, query_context); + StorageInMemoryMetadata future_metadata = getInMemoryMetadata(); + params.apply(future_metadata, query_context); - changeSettings(metadata.settings_changes, table_lock_holder); + changeSettings(future_metadata.settings_changes, table_lock_holder); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id, future_metadata); return; } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index b82b387a623..8e136995917 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -540,7 +540,7 @@ protected: bool attach, const StorageID & table_id_, const String & relative_data_path_, - const StorageInMemoryMetadata & metadata, + const StorageInMemoryMetadata & metadata_, Context & context_, const String & date_column_name, const MergingParams & merging_params_, From 0d375e45223e41432b5539ce48019ee158843b1e Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 8 Jun 2020 21:49:54 +0300 Subject: [PATCH 036/329] Fix constraints --- src/Storages/AlterCommands.cpp | 3 ++- src/Storages/MergeTree/MergeTreeData.cpp | 8 ++++---- src/Storages/MergeTree/registerStorageMergeTree.cpp | 9 +++++++++ .../0_stateless/01277_alter_rename_column_constraint.sql | 6 +++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 13da6db8ed3..33a62d9bd1c 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -327,7 +327,8 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con } else if (type == COMMENT_COLUMN) { - metadata.columns.modify(column_name, [&](ColumnDescription & column) { column.comment = *comment; }); + metadata.columns.modify(column_name, + [&](ColumnDescription & column) { column.comment = *comment; }); } else if (type == ADD_INDEX) { diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 593498d5e84..fe13446def7 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -404,10 +404,10 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool { /// Other parts of metadata initialized is separate methods setColumns(std::move(metadata.columns)); - setSecondaryIndices(metadata.secondary_indices); - setConstraints(metadata.constraints); - setSortingKey(new_sorting_key); - setPrimaryKey(new_primary_key); + setSecondaryIndices(std::move(metadata.secondary_indices)); + setConstraints(std::move(metadata.constraints)); + setSortingKey(std::move(new_sorting_key)); + setPrimaryKey(std::move(new_primary_key)); } } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index d61418edfe7..50775a04255 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -506,6 +506,11 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (args.storage_def->primary_key) metadata.primary_key = KeyDescription::getKeyFromAST(args.storage_def->primary_key->ptr(), metadata.columns, args.context); + else + { + metadata.primary_key = metadata.sorting_key; + metadata.primary_key.definition_ast = nullptr; + } if (args.storage_def->sample_by) metadata.sampling_key = KeyDescription::getKeyFromAST(args.storage_def->sample_by->ptr(), metadata.columns, args.context); @@ -521,6 +526,10 @@ static StoragePtr create(const StorageFactory::Arguments & args) for (auto & index : args.query.columns_list->indices->children) metadata.secondary_indices.push_back(IndexDescription::getIndexFromAST(index, args.columns, args.context)); + if (args.query.columns_list && args.query.columns_list->constraints) + for (auto & constraint : args.query.columns_list->constraints->children) + metadata.constraints.constraints.push_back(constraint); + storage_settings->loadFromQuery(*args.storage_def); if (args.storage_def->settings) diff --git a/tests/queries/0_stateless/01277_alter_rename_column_constraint.sql b/tests/queries/0_stateless/01277_alter_rename_column_constraint.sql index 72fbb045601..b9d5030239d 100644 --- a/tests/queries/0_stateless/01277_alter_rename_column_constraint.sql +++ b/tests/queries/0_stateless/01277_alter_rename_column_constraint.sql @@ -15,7 +15,7 @@ PARTITION BY date ORDER BY key; INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(9); -INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(9); ; --{serverError 469} +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(9); --{serverError 469} SELECT * FROM table_for_rename ORDER BY key; @@ -26,7 +26,7 @@ SELECT * FROM table_for_rename ORDER BY key; SELECT '-- insert after rename --'; INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(10, 10); -INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(10, 10); ; --{serverError 469} +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(10, 10); --{serverError 469} SELECT * FROM table_for_rename ORDER BY key; SELECT '-- rename columns back --'; @@ -37,7 +37,7 @@ SELECT * FROM table_for_rename ORDER BY key; SELECT '-- insert after rename column --'; INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(20,10); -INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number + 2) from numbers(20, 10); ; --{serverError 469} +INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number + 2) from numbers(20, 10); --{serverError 469} SELECT * FROM table_for_rename ORDER BY key; DROP TABLE IF EXISTS table_for_rename; From 75075a7b2bb493ee9306ffe1e099e182249560a4 Mon Sep 17 00:00:00 2001 From: hexiaoting <“hewenting_ict@163.com”> Date: Tue, 9 Jun 2020 10:39:37 +0800 Subject: [PATCH 037/329] Add DISTINCT keyword when show clusters --- src/Interpreters/InterpreterShowTablesQuery.cpp | 2 +- tests/queries/0_stateless/01293_show_clusters.reference | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 10447e52464..4b0d4c21ad1 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -37,7 +37,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() if (query.clusters) { std::stringstream rewritten_query; - rewritten_query << "SELECT cluster FROM system.clusters"; + rewritten_query << "SELECT DISTINCT cluster FROM system.clusters"; if (!query.like.empty()) { diff --git a/tests/queries/0_stateless/01293_show_clusters.reference b/tests/queries/0_stateless/01293_show_clusters.reference index b25a9a4d174..85a14155529 100644 --- a/tests/queries/0_stateless/01293_show_clusters.reference +++ b/tests/queries/0_stateless/01293_show_clusters.reference @@ -1,11 +1,8 @@ test_cluster_two_shards -test_cluster_two_shards -test_cluster_two_shards_localhost test_cluster_two_shards_localhost test_shard_localhost test_shard_localhost[1] test_shard_localhost_secure test_unavailable_shard -test_unavailable_shard test_cluster_two_shards test_shard_localhost 1 1 1 localhost ::1 9000 1 default 0 0 From 6f673b0981c1b5c1a90df917d1f8d479309a5b0f Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 9 Jun 2020 19:33:28 +0300 Subject: [PATCH 038/329] Fix comments bug --- src/Storages/IStorage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 2ad89f559b5..31dd87a7ce4 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -382,8 +382,8 @@ void IStorage::alter( auto table_id = getStorageID(); StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); params.apply(old_metadata, context); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata); - setColumns(std::move(metadata.columns)); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, old_metadata); + setColumns(std::move(old_metadata.columns)); } From 4c2e0561f501e7d2c8593aabf80495a1a0386529 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 9 Jun 2020 20:21:38 +0300 Subject: [PATCH 039/329] Fix primary key update --- src/Storages/MergeTree/MergeTreeData.cpp | 28 ++++++++++++------------ src/Storages/MergeTree/MergeTreeData.h | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 8ca125eff64..0f142089bc2 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -275,22 +275,22 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam } } -void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool only_check, bool attach) +void MergeTreeData::setProperties(const StorageInMemoryMetadata & new_metadata, bool only_check, bool attach) { - KeyDescription new_primary_key = metadata.primary_key; + KeyDescription new_primary_key = new_metadata.primary_key; - if (!metadata.sorting_key.definition_ast) + if (!new_metadata.sorting_key.definition_ast) throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS); KeyDescription new_sorting_key; if (merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) new_sorting_key = KeyDescription::getKeyFromAST( - metadata.sorting_key.definition_ast, - metadata.columns, + new_metadata.sorting_key.definition_ast, + new_metadata.columns, global_context, std::make_shared(merging_params.version_column)); else - new_sorting_key = metadata.sorting_key; + new_sorting_key = new_metadata.sorting_key; /// Primary key not defined at all if (new_primary_key.definition_ast == nullptr) @@ -299,7 +299,7 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool /// because in merge tree code we sometimes chech, that our primary key /// is fake (copied from sorting key, i.e. isPrimaryKeyDefined() == /// false, but hasSortingKey() == true) - new_primary_key = metadata.sorting_key; + new_primary_key = new_metadata.sorting_key; new_primary_key.definition_ast = nullptr; } @@ -330,7 +330,7 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool } } - auto all_columns = metadata.columns.getAllPhysical(); + auto all_columns = new_metadata.columns.getAllPhysical(); /// Order by check AST if (hasSortingKey() && only_check) @@ -372,7 +372,7 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool "added to the sorting key. You can add expressions that use only the newly added columns", ErrorCodes::BAD_ARGUMENTS); - if (metadata.columns.getDefaults().count(col)) + if (new_metadata.columns.getDefaults().count(col)) throw Exception("Newly added column " + col + " has a default expression, so adding " "expressions that use it to the sorting key is forbidden", ErrorCodes::BAD_ARGUMENTS); @@ -380,11 +380,11 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool } } - if (!metadata.secondary_indices.empty()) + if (!new_metadata.secondary_indices.empty()) { std::unordered_set indices_names; - for (const auto & index : metadata.secondary_indices) + for (const auto & index : new_metadata.secondary_indices) { MergeTreeIndexFactory::instance().validate(index, attach); @@ -403,9 +403,9 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool if (!only_check) { /// Other parts of metadata initialized is separate methods - setColumns(std::move(metadata.columns)); - setSecondaryIndices(std::move(metadata.secondary_indices)); - setConstraints(std::move(metadata.constraints)); + setColumns(std::move(new_metadata.columns)); + setSecondaryIndices(std::move(new_metadata.secondary_indices)); + setConstraints(std::move(new_metadata.constraints)); setSortingKey(std::move(new_sorting_key)); setPrimaryKey(std::move(new_primary_key)); } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 8b24bea7830..4080a898106 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -780,7 +780,7 @@ protected: /// The same for clearOldTemporaryDirectories. std::mutex clear_old_temporary_directories_mutex; - void setProperties(const StorageInMemoryMetadata & metadata, bool only_check = false, bool attach = false); + void setProperties(const StorageInMemoryMetadata & new_metadata, bool only_check = false, bool attach = false); void initPartitionKey(const KeyDescription & new_partition_key); From 936cc9d57376d1e307a75cb880b93dab2c352fe2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 9 Jun 2020 20:22:03 +0300 Subject: [PATCH 040/329] Update key --- src/Storages/AlterCommands.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 33a62d9bd1c..5652d1717ec 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -313,6 +313,9 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con column.default_desc.expression = default_expression; } }); + + if (metadata.sorting_key.sample_block.has(column_name)) + metadata.sorting_key = KeyDescription::getKeyFromAST(metadata.sorting_key.definition_ast, metadata.columns, context); } else if (type == MODIFY_ORDER_BY) { From d2fcf5aea5421508f5b9ce3dec1b282dc9f6ecf1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 9 Jun 2020 20:28:29 +0300 Subject: [PATCH 041/329] Fixes for gcc --- src/Storages/MergeTree/MergeTreeData.cpp | 16 ++++++++-------- src/Storages/MergeTree/MergeTreeData.h | 2 +- src/Storages/StorageBuffer.cpp | 8 ++++---- src/Storages/StorageDistributed.cpp | 8 ++++---- src/Storages/StorageMergeTree.cpp | 24 ++++++++++++------------ src/Storages/StorageNull.cpp | 8 ++++---- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0f142089bc2..dd70e7b2b5c 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -118,7 +118,7 @@ const char * DELETE_ON_DESTROY_MARKER_PATH = "delete-on-destroy.txt"; MergeTreeData::MergeTreeData( const StorageID & table_id_, const String & relative_data_path_, - const StorageInMemoryMetadata & metadata, + const StorageInMemoryMetadata & metadata_, Context & context_, const String & date_column_name, const MergingParams & merging_params_, @@ -142,21 +142,21 @@ MergeTreeData::MergeTreeData( if (relative_data_path.empty()) throw Exception("MergeTree storages require data path", ErrorCodes::INCORRECT_FILE_NAME); - setSettingsChanges(metadata.settings_changes); + setSettingsChanges(metadata_.settings_changes); const auto settings = getSettings(); - setProperties(metadata, /*only_check*/ false, attach); + setProperties(metadata_, /*only_check*/ false, attach); /// NOTE: using the same columns list as is read when performing actual merges. merging_params.check(getColumns().getAllPhysical()); - if (metadata.sampling_key.definition_ast != nullptr) + if (metadata_.sampling_key.definition_ast != nullptr) { const auto & pk_sample_block = getPrimaryKey().sample_block; - if (!pk_sample_block.has(metadata.sampling_key.column_names[0]) && !attach + if (!pk_sample_block.has(metadata_.sampling_key.column_names[0]) && !attach && !settings->compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility. throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS); - setSamplingKey(metadata.sampling_key); + setSamplingKey(metadata_.sampling_key); } MergeTreeDataFormatVersion min_format_version(0); @@ -181,11 +181,11 @@ MergeTreeData::MergeTreeData( else { is_custom_partitioned = true; - initPartitionKey(metadata.partition_key); + initPartitionKey(metadata_.partition_key); min_format_version = MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING; } - setTTLExpressions(metadata.columns, metadata.table_ttl); + setTTLExpressions(metadata_.columns, metadata_.table_ttl); /// format_file always contained on any data path PathWithDisk version_file; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 4080a898106..0e2e9c71324 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -323,7 +323,7 @@ public: /// attach - whether the existing table is attached or the new table is created. MergeTreeData(const StorageID & table_id_, const String & relative_data_path_, - const StorageInMemoryMetadata & metadata, + const StorageInMemoryMetadata & metadata_, Context & context_, const String & date_column_name, const MergingParams & merging_params_, diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 2d8c3fd9a2f..1f2c05bf7ad 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -775,10 +775,10 @@ void StorageBuffer::alter(const AlterCommands & params, const Context & context, /// So that no blocks of the old structure remain. optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context); - StorageInMemoryMetadata metadata = getInMemoryMetadata(); - params.apply(metadata, context); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata); - setColumns(std::move(metadata.columns)); + StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); + params.apply(new_metadata, context); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata); + setColumns(std::move(new_metadata.columns)); } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index d80fee1e4dc..7e7460a013f 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -559,10 +559,10 @@ void StorageDistributed::alter(const AlterCommands & params, const Context & con auto table_id = getStorageID(); checkAlterIsPossible(params, context.getSettingsRef()); - StorageInMemoryMetadata metadata = getInMemoryMetadata(); - params.apply(metadata, context); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata); - setColumns(std::move(metadata.columns)); + StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); + params.apply(new_metadata, context); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata); + setColumns(std::move(new_metadata.columns)); } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 8ccd1712232..76f752abb68 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -52,7 +52,7 @@ namespace ActionLocks StorageMergeTree::StorageMergeTree( const StorageID & table_id_, const String & relative_data_path_, - const StorageInMemoryMetadata & metadata, + const StorageInMemoryMetadata & metadata_, bool attach, Context & context_, const String & date_column_name, @@ -62,7 +62,7 @@ StorageMergeTree::StorageMergeTree( : MergeTreeData( table_id_, relative_data_path_, - metadata, + metadata_, context_, date_column_name, merging_params_, @@ -256,20 +256,20 @@ void StorageMergeTree::alter( { auto table_id = getStorageID(); - StorageInMemoryMetadata metadata = getInMemoryMetadata(); - auto maybe_mutation_commands = commands.getMutationCommands(metadata, context.getSettingsRef().materialize_ttl_after_modify, context); + StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); + auto maybe_mutation_commands = commands.getMutationCommands(new_metadata, context.getSettingsRef().materialize_ttl_after_modify, context); String mutation_file_name; Int64 mutation_version = -1; - commands.apply(metadata, context); + commands.apply(new_metadata, context); - /// This alter can be performed at metadata level only + /// This alter can be performed at new_metadata level only if (commands.isSettingsAlter()) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); - changeSettings(metadata.settings_changes, table_lock_holder); + changeSettings(new_metadata.settings_changes, table_lock_holder); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata); } else { @@ -278,13 +278,13 @@ void StorageMergeTree::alter( auto merges_block = getActionLock(ActionLocks::PartsMerge); lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); - changeSettings(metadata.settings_changes, table_lock_holder); + changeSettings(new_metadata.settings_changes, table_lock_holder); /// Reinitialize primary key because primary key column types might have changed. - setProperties(metadata); + setProperties(new_metadata); - setTTLExpressions(metadata.columns, metadata.table_ttl); + setTTLExpressions(new_metadata.columns, new_metadata.table_ttl); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata); if (!maybe_mutation_commands.empty()) mutation_version = startMutation(maybe_mutation_commands, mutation_file_name); diff --git a/src/Storages/StorageNull.cpp b/src/Storages/StorageNull.cpp index c29562325fa..b6e4605530d 100644 --- a/src/Storages/StorageNull.cpp +++ b/src/Storages/StorageNull.cpp @@ -51,10 +51,10 @@ void StorageNull::alter( lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto table_id = getStorageID(); - StorageInMemoryMetadata metadata = getInMemoryMetadata(); - params.apply(metadata, context); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata); - setColumns(std::move(metadata.columns)); + StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); + params.apply(new_metadata, context); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata); + setColumns(std::move(new_metadata.columns)); } } From 4c77ba664a95e7c474af18a79a09a5733f09a740 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 9 Jun 2020 20:42:04 +0300 Subject: [PATCH 042/329] Better assignment --- src/Storages/IndicesDescription.cpp | 3 +++ src/Storages/KeyDescription.cpp | 3 +++ src/Storages/SelectQueryDescription.cpp | 3 +++ src/Storages/StorageInMemoryMetadata.cpp | 3 +++ src/Storages/TTLDescription.cpp | 3 +++ 5 files changed, 15 insertions(+) diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index 6f8406d9a3f..d59aef2ecaa 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -36,6 +36,9 @@ IndexDescription::IndexDescription(const IndexDescription & other) IndexDescription & IndexDescription::operator=(const IndexDescription & other) { + if (&other == this) + return *this; + if (other.definition_ast) definition_ast = other.definition_ast->clone(); else diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index e59693f5343..5d44ced78db 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -22,6 +22,9 @@ KeyDescription::KeyDescription(const KeyDescription & other) KeyDescription & KeyDescription::operator=(const KeyDescription & other) { + if (&other == this) + return *this; + if (other.definition_ast) definition_ast = other.definition_ast->clone(); else diff --git a/src/Storages/SelectQueryDescription.cpp b/src/Storages/SelectQueryDescription.cpp index 3d2ba27a62c..87ba4ce74a9 100644 --- a/src/Storages/SelectQueryDescription.cpp +++ b/src/Storages/SelectQueryDescription.cpp @@ -24,6 +24,9 @@ SelectQueryDescription::SelectQueryDescription(const SelectQueryDescription & ot SelectQueryDescription & SelectQueryDescription::SelectQueryDescription::operator=(const SelectQueryDescription & other) { + if (&other == this) + return *this; + select_table_id = other.select_table_id; if (other.select_query) select_query = other.select_query->clone(); diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 0f8b88e691b..e83a41a3877 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -30,6 +30,9 @@ StorageInMemoryMetadata::StorageInMemoryMetadata(const StorageInMemoryMetadata & StorageInMemoryMetadata & StorageInMemoryMetadata::operator=(const StorageInMemoryMetadata & other) { + if (&other == this) + return *this; + columns = other.columns; secondary_indices = other.secondary_indices; constraints = other.constraints; diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index ac77bd51b69..e241b7676a0 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -72,6 +72,9 @@ TTLDescription::TTLDescription(const TTLDescription & other) TTLDescription & TTLDescription::operator=(const TTLDescription & other) { + if (&other == this) + return *this; + mode = other.mode; if (other.expression_ast) expression_ast = other.expression_ast->clone(); From 8a78cffe5c3e8b9ac2e26e0731c40306cad4c9e1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Jun 2020 21:11:08 +0300 Subject: [PATCH 043/329] Force table creation on SYSTEM FLUSH LOGS --- src/Interpreters/InterpreterSystemQuery.cpp | 12 +++--- src/Interpreters/SystemLog.h | 14 +++++-- .../test_SYSTEM_FLUSH_LOGS/test.py | 38 +++++++++++++++++++ 3 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 tests/integration/test_SYSTEM_FLUSH_LOGS/test.py diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 9ebdb155643..1480651b4b6 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -301,12 +301,12 @@ BlockIO InterpreterSystemQuery::execute() case Type::FLUSH_LOGS: context.checkAccess(AccessType::SYSTEM_FLUSH_LOGS); executeCommandsAndThrowIfError( - [&] () { if (auto query_log = context.getQueryLog()) query_log->flush(); }, - [&] () { if (auto part_log = context.getPartLog("")) part_log->flush(); }, - [&] () { if (auto query_thread_log = context.getQueryThreadLog()) query_thread_log->flush(); }, - [&] () { if (auto trace_log = context.getTraceLog()) trace_log->flush(); }, - [&] () { if (auto text_log = context.getTextLog()) text_log->flush(); }, - [&] () { if (auto metric_log = context.getMetricLog()) metric_log->flush(); } + [&] () { if (auto query_log = context.getQueryLog()) query_log->flush(true); }, + [&] () { if (auto part_log = context.getPartLog("")) part_log->flush(true); }, + [&] () { if (auto query_thread_log = context.getQueryThreadLog()) query_thread_log->flush(true); }, + [&] () { if (auto trace_log = context.getTraceLog()) trace_log->flush(true); }, + [&] () { if (auto text_log = context.getTextLog()) text_log->flush(true); }, + [&] () { if (auto metric_log = context.getMetricLog()) metric_log->flush(true); } ); break; case Type::STOP_LISTEN_QUERIES: diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index dd2f815ce92..1f889d53aec 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -76,7 +76,8 @@ class ISystemLog public: virtual String getName() = 0; virtual ASTPtr getCreateTableQuery() = 0; - virtual void flush() = 0; + //// force -- force table creation (used for SYSTEM FLUSH LOGS) + virtual void flush(bool force = false) = 0; virtual void prepareTable() = 0; virtual void startup() = 0; virtual void shutdown() = 0; @@ -133,7 +134,7 @@ public: void stopFlushThread(); /// Flush data in the buffer to disk - void flush() override; + void flush(bool force = false) override; /// Start the background thread. void startup() override; @@ -166,6 +167,8 @@ private: /* Data shared between callers of add()/flush()/shutdown(), and the saving thread */ std::mutex mutex; + /* prepareTable() guard */ + std::mutex prepare_mutex; // Queue is bounded. But its size is quite large to not block in all normal cases. std::vector queue; // An always-incrementing index of the first message currently in the queue. @@ -272,13 +275,16 @@ void SystemLog::add(const LogElement & element) template -void SystemLog::flush() +void SystemLog::flush(bool force) { std::unique_lock lock(mutex); if (is_shutdown) return; + if (force) + prepareTable(); + const uint64_t queue_end = queue_front_index + queue.size(); if (requested_flush_before < queue_end) @@ -429,6 +435,8 @@ void SystemLog::flushImpl(const std::vector & to_flush, template void SystemLog::prepareTable() { + std::unique_lock prepare_lock(prepare_mutex); + String description = table_id.getNameForLogs(); table = DatabaseCatalog::instance().tryGetTable(table_id, context); diff --git a/tests/integration/test_SYSTEM_FLUSH_LOGS/test.py b/tests/integration/test_SYSTEM_FLUSH_LOGS/test.py new file mode 100644 index 00000000000..2329094e150 --- /dev/null +++ b/tests/integration/test_SYSTEM_FLUSH_LOGS/test.py @@ -0,0 +1,38 @@ +# pylint: disable=line-too-long +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name + +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node_default') + +system_logs = [ + # disabled by default + ('system.part_log', 0), + ('system.text_log', 0), + + # enabled by default + ('system.query_log', 1), + ('system.query_thread_log', 1), + ('system.trace_log', 1), + ('system.metric_log', 1), +] + +@pytest.fixture(scope='module') +def start_cluster(): + try: + cluster.start() + node.query('SYSTEM FLUSH LOGS') + yield cluster + finally: + cluster.shutdown() + +@pytest.mark.parametrize('table,exists', system_logs) +def test_system_logs(start_cluster, table, exists): + q = 'SELECT * FROM {}'.format(table) + if exists: + node.query(q) + else: + assert "Table {} doesn't exist".format(table) in node.query_and_get_error(q) From 75977bd939887daa55f320951b431f4392a0acb6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Jun 2020 00:22:01 +0300 Subject: [PATCH 044/329] Gcc fixed --- src/Storages/MergeTree/MergeTreeData.cpp | 12 ++++++------ src/Storages/StorageMaterializedView.cpp | 10 +++++----- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index dd70e7b2b5c..50971bbc881 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1273,9 +1273,9 @@ bool isMetadataOnlyConversion(const IDataType * from, const IDataType * to) void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) { /// Check that needed transformations can be applied to the list of columns without considering type conversions. - StorageInMemoryMetadata metadata = getInMemoryMetadata(); - commands.apply(metadata, global_context); - if (getSecondaryIndices().empty() && !metadata.secondary_indices.empty() && + StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); + commands.apply(new_metadata, global_context); + if (getSecondaryIndices().empty() && !new_metadata.secondary_indices.empty() && !settings.allow_experimental_data_skipping_indices) throw Exception("You must set the setting `allow_experimental_data_skipping_indices` to 1 " \ "before using data skipping indices.", ErrorCodes::BAD_ARGUMENTS); @@ -1365,15 +1365,15 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S } } - setProperties(metadata, /* only_check = */ true); + setProperties(new_metadata, /* only_check = */ true); - setTTLExpressions(metadata.columns, metadata.table_ttl, /* only_check = */ true); + setTTLExpressions(new_metadata.columns, new_metadata.table_ttl, /* only_check = */ true); if (hasSettingsChanges()) { const auto & current_changes = getSettingsChanges()->as().changes; - const auto & new_changes = metadata.settings_changes->as().changes; + const auto & new_changes = new_metadata.settings_changes->as().changes; for (const auto & changed_setting : new_changes) { if (MergeTreeSettings::findIndex(changed_setting.name) == MergeTreeSettings::npos) diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index a7a59f0e9b9..08984a6e1f3 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -193,13 +193,13 @@ void StorageMaterializedView::alter( { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto table_id = getStorageID(); - StorageInMemoryMetadata metadata = getInMemoryMetadata(); - params.apply(metadata, context); + StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); + params.apply(new_metadata, context); /// start modify query if (context.getSettingsRef().allow_experimental_alter_materialized_view_structure) { - const auto & new_select = metadata.select; + const auto & new_select = new_metadata.select; const auto & old_select = getSelectQuery(); DatabaseCatalog::instance().updateDependency(old_select.select_table_id, table_id, new_select.select_table_id, table_id); @@ -208,8 +208,8 @@ void StorageMaterializedView::alter( } /// end modify query - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata); - setColumns(std::move(metadata.columns)); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata); + setColumns(std::move(new_metadata.columns)); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f4941a05bf2..8efe22e03f9 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -415,12 +415,12 @@ void StorageReplicatedMergeTree::createTableIfNotExists() zookeeper->createAncestors(zookeeper_path); /// We write metadata of table so that the replicas can check table parameters with them. - String metadata = ReplicatedMergeTreeTableMetadata(*this).toString(); + String metadata_str = ReplicatedMergeTreeTableMetadata(*this).toString(); Coordination::Requests ops; ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path, "", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/metadata", metadata, + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/metadata", metadata_str, zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/columns", getColumns().toString(), zkutil::CreateMode::Persistent)); From ec933d9d03657edd468c11e2a5f35d4dd219e1b6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Jun 2020 12:09:51 +0300 Subject: [PATCH 045/329] Better naming --- src/Storages/AlterCommands.cpp | 27 ++++++++++++++-- src/Storages/IStorage.cpp | 8 ++--- src/Storages/IStorage.h | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 32 ++++++++----------- src/Storages/MergeTree/MergeTreeData.h | 3 +- .../MergeTree/registerStorageMergeTree.cpp | 7 ++++ src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 22 ++++++------- tests/queries/0_stateless/00933_alter_ttl.sql | 3 +- 9 files changed, 65 insertions(+), 40 deletions(-) diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 5652d1717ec..4755ad0fd35 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -314,8 +314,6 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con } }); - if (metadata.sorting_key.sample_block.has(column_name)) - metadata.sorting_key = KeyDescription::getKeyFromAST(metadata.sorting_key.definition_ast, metadata.columns, context); } else if (type == MODIFY_ORDER_BY) { @@ -714,6 +712,31 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, const Context & co if (!command.ignore) command.apply(metadata_copy, context); + /// Changes in columns may lead to changes in keys expression + metadata_copy.sorting_key = KeyDescription::getKeyFromAST(metadata_copy.sorting_key.definition_ast, metadata_copy.columns, context); + + if (metadata_copy.primary_key.definition_ast != nullptr) + { + metadata_copy.primary_key = KeyDescription::getKeyFromAST(metadata_copy.primary_key.definition_ast, metadata_copy.columns, context); + } + else + { + metadata_copy.primary_key = metadata_copy.sorting_key; + metadata_copy.primary_key.definition_ast = nullptr; + } + + /// Changes in columns may lead to changes in TTL expressions + auto column_ttl_asts = metadata_copy.columns.getColumnTTLs(); + for (const auto & [name, ast] : column_ttl_asts) + { + auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, metadata_copy.columns, context, metadata_copy.primary_key); + metadata_copy.column_ttls_by_name[name] = new_ttl_entry; + } + + if (metadata_copy.table_ttl.definition_ast != nullptr) + metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + metadata_copy.table_ttl.definition_ast, metadata_copy.columns, context, metadata_copy.primary_key); + metadata = std::move(metadata_copy); } diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 31dd87a7ce4..83c95d9a20f 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -380,10 +380,10 @@ void IStorage::alter( { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); auto table_id = getStorageID(); - StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); - params.apply(old_metadata, context); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, old_metadata); - setColumns(std::move(old_metadata.columns)); + StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); + params.apply(new_metadata, context); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata); + setColumns(std::move(new_metadata.columns)); } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 82012eacc03..c06b969ebf5 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -154,6 +154,7 @@ public: /// thread-unsafe part. lockStructure must be acquired void setSettingsChanges(const ASTPtr & settings_changes_); bool hasSettingsChanges() const { return metadata.settings_changes != nullptr; } + /// Select query for *View storages. const SelectQueryDescription & getSelectQuery() const; void setSelectQuery(const SelectQueryDescription & select_); bool hasSelectQuery() const; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 50971bbc881..4a65d05cc8e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -185,7 +185,7 @@ MergeTreeData::MergeTreeData( min_format_version = MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING; } - setTTLExpressions(metadata_.columns, metadata_.table_ttl); + setTTLExpressions(metadata_); /// format_file always contained on any data path PathWithDisk version_file; @@ -296,9 +296,9 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & new_metadata, if (new_primary_key.definition_ast == nullptr) { /// We copy sorting key, and restore definition_ast to empty value, - /// because in merge tree code we sometimes chech, that our primary key - /// is fake (copied from sorting key, i.e. isPrimaryKeyDefined() == - /// false, but hasSortingKey() == true) + /// because in merge tree code we chech, that our primary key is fake + /// (copied from sorting key, i.e. isPrimaryKeyDefined() == false, but + /// hasSortingKey() == true) new_primary_key = new_metadata.sorting_key; new_primary_key.definition_ast = nullptr; } @@ -522,14 +522,11 @@ void MergeTreeData::initPartitionKey(const KeyDescription & new_partition_key) /// Todo replace columns with TTL for columns -void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns, - const TTLTableDescription & new_table_ttl, bool only_check) +void MergeTreeData::setTTLExpressions(const StorageInMemoryMetadata & new_metadata, bool only_check) { - auto new_column_ttls_asts = new_columns.getColumnTTLs(); + auto new_column_ttls = new_metadata.column_ttls_by_name; - TTLColumnsDescription new_column_ttl_by_name = getColumnTTLs(); - - if (!new_column_ttls_asts.empty()) + if (!new_column_ttls.empty()) { NameSet columns_ttl_forbidden; @@ -541,20 +538,18 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns, for (const auto & col : getColumnsRequiredForSortingKey()) columns_ttl_forbidden.insert(col); - for (const auto & [name, ast] : new_column_ttls_asts) + for (const auto & [name, ttl_description] : new_column_ttls) { if (columns_ttl_forbidden.count(name)) throw Exception("Trying to set TTL for key column " + name, ErrorCodes::ILLEGAL_COLUMN); - else - { - auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_columns, global_context, getPrimaryKey()); - new_column_ttl_by_name[name] = new_ttl_entry; - } } + if (!only_check) - setColumnTTLs(new_column_ttl_by_name); + setColumnTTLs(new_column_ttls); } + auto new_table_ttl = new_metadata.table_ttl; + if (new_table_ttl.definition_ast) { for (const auto & move_ttl : new_table_ttl.move_ttl) @@ -570,7 +565,6 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns, } } - if (!only_check) { auto move_ttl_entries_lock = std::lock_guard(move_ttl_entries_mutex); @@ -1367,7 +1361,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S setProperties(new_metadata, /* only_check = */ true); - setTTLExpressions(new_metadata.columns, new_metadata.table_ttl, /* only_check = */ true); + setTTLExpressions(new_metadata, /* only_check = */ true); if (hasSettingsChanges()) { diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 0e2e9c71324..a239f8cc0d7 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -784,8 +784,7 @@ protected: void initPartitionKey(const KeyDescription & new_partition_key); - void setTTLExpressions(const ColumnsDescription & columns, - const TTLTableDescription & new_table_ttl, bool only_check = false); + void setTTLExpressions(const StorageInMemoryMetadata & new_metadata, bool only_check = false); void checkStoragePolicy(const StoragePolicyPtr & new_storage_policy) const; diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 50775a04255..90eee680d65 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -530,6 +530,13 @@ static StoragePtr create(const StorageFactory::Arguments & args) for (auto & constraint : args.query.columns_list->constraints->children) metadata.constraints.constraints.push_back(constraint); + auto column_ttl_asts = args.columns.getColumnTTLs(); + for (const auto & [name, ast] : column_ttl_asts) + { + auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, args.columns, args.context, metadata.primary_key); + metadata.column_ttls_by_name[name] = new_ttl_entry; + } + storage_settings->loadFromQuery(*args.storage_def); if (args.storage_def->settings) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 76f752abb68..2a7efa164d4 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -282,7 +282,7 @@ void StorageMergeTree::alter( /// Reinitialize primary key because primary key column types might have changed. setProperties(new_metadata); - setTTLExpressions(new_metadata.columns, new_metadata.table_ttl); + setTTLExpressions(new_metadata); DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8efe22e03f9..42adf4d2b45 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -472,9 +472,9 @@ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff) { - StorageInMemoryMetadata current_metadata = getInMemoryMetadata(); - if (new_columns != current_metadata.columns) - current_metadata.columns = new_columns; + StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); + if (new_columns != new_metadata.columns) + new_metadata.columns = new_columns; if (!metadata_diff.empty()) { @@ -492,37 +492,37 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column tuple->arguments->children = new_sorting_key_expr_list->children; order_by_ast = tuple; } - current_metadata.sorting_key = KeyDescription::getKeyFromAST(order_by_ast, current_metadata.columns, global_context); + new_metadata.sorting_key = KeyDescription::getKeyFromAST(order_by_ast, new_metadata.columns, global_context); if (!isPrimaryKeyDefined()) { /// Primary and sorting key become independent after this ALTER so we have to /// save the old ORDER BY expression as the new primary key. - current_metadata.primary_key = getSortingKey(); + new_metadata.primary_key = getSortingKey(); } } if (metadata_diff.skip_indices_changed) - current_metadata.secondary_indices = IndicesDescription::parse(metadata_diff.new_skip_indices, new_columns, global_context); + new_metadata.secondary_indices = IndicesDescription::parse(metadata_diff.new_skip_indices, new_columns, global_context); if (metadata_diff.constraints_changed) - current_metadata.constraints = ConstraintsDescription::parse(metadata_diff.new_constraints); + new_metadata.constraints = ConstraintsDescription::parse(metadata_diff.new_constraints); if (metadata_diff.ttl_table_changed) { ParserTTLExpressionList parser; auto ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - current_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl_for_table_ast, current_metadata.columns, global_context, current_metadata.primary_key); + new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl_for_table_ast, new_metadata.columns, global_context, new_metadata.primary_key); } } auto table_id = getStorageID(); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(global_context, table_id, current_metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(global_context, table_id, new_metadata); /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. - setProperties(current_metadata); - setTTLExpressions(new_columns, current_metadata.table_ttl); + setProperties(new_metadata); + setTTLExpressions(new_metadata); } diff --git a/tests/queries/0_stateless/00933_alter_ttl.sql b/tests/queries/0_stateless/00933_alter_ttl.sql index d3298b3fbe1..4e0fde00952 100644 --- a/tests/queries/0_stateless/00933_alter_ttl.sql +++ b/tests/queries/0_stateless/00933_alter_ttl.sql @@ -21,6 +21,7 @@ drop table if exists ttl; create table ttl (d Date, a Int) engine = MergeTree order by tuple() partition by toDayOfMonth(d); alter table ttl modify column a Int ttl d + interval 1 day; desc table ttl; -alter table ttl modify column d Int ttl d + interval 1 day; -- { serverError 524 } +alter table ttl modify column d Int ttl d + interval 1 day; -- { serverError 43 } +alter table ttl modify column d DateTime ttl d + interval 1 day; -- { serverError 524 } drop table if exists ttl; From 8be957ecb5d4869b311629d741d5b56ef63510cf Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Jun 2020 14:16:31 +0300 Subject: [PATCH 046/329] Better checks around metadata --- programs/copier/Internals.cpp | 5 +- src/Storages/AlterCommands.cpp | 10 +- src/Storages/IStorage.cpp | 2 +- src/Storages/IStorage.h | 2 +- src/Storages/KeyDescription.cpp | 12 ++- src/Storages/KeyDescription.h | 9 +- src/Storages/MergeTree/MergeTreeData.cpp | 100 ++++++------------ src/Storages/MergeTree/MergeTreeData.h | 15 ++- .../MergeTree/registerStorageMergeTree.cpp | 15 ++- src/Storages/StorageBuffer.cpp | 2 +- src/Storages/StorageBuffer.h | 2 +- src/Storages/StorageDistributed.cpp | 2 +- src/Storages/StorageDistributed.h | 2 +- src/Storages/StorageMaterializedView.cpp | 2 +- src/Storages/StorageMaterializedView.h | 2 +- src/Storages/StorageMerge.cpp | 2 +- src/Storages/StorageMerge.h | 2 +- src/Storages/StorageNull.cpp | 2 +- src/Storages/StorageNull.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- ...213_alter_rename_primary_key_zookeeper.sql | 12 +-- 21 files changed, 94 insertions(+), 110 deletions(-) diff --git a/programs/copier/Internals.cpp b/programs/copier/Internals.cpp index 0613381a763..518395e3b7d 100644 --- a/programs/copier/Internals.cpp +++ b/programs/copier/Internals.cpp @@ -1,5 +1,6 @@ #include "Internals.h" #include +#include namespace DB { @@ -184,9 +185,9 @@ Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast) const auto sorting_key_ast = extractOrderBy(storage_ast); const auto primary_key_ast = extractPrimaryKey(storage_ast); - const auto sorting_key_expr_list = MergeTreeData::extractKeyExpressionList(sorting_key_ast); + const auto sorting_key_expr_list = extractKeyExpressionList(sorting_key_ast); const auto primary_key_expr_list = primary_key_ast - ? MergeTreeData::extractKeyExpressionList(primary_key_ast) : sorting_key_expr_list->clone(); + ? extractKeyExpressionList(primary_key_ast) : sorting_key_expr_list->clone(); /// Maybe we have to handle VersionedCollapsing engine separately. But in our case in looks pointless. diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 4755ad0fd35..52f34d1fdd5 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -324,7 +324,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con metadata.primary_key = metadata.sorting_key; } - metadata.sorting_key = KeyDescription::getKeyFromAST(order_by, metadata.columns, context); + metadata.sorting_key = KeyDescription::getKeyFromAST(order_by, metadata.columns, context, metadata.sorting_key.additional_key_column); } else if (type == COMMENT_COLUMN) { @@ -713,7 +713,11 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, const Context & co command.apply(metadata_copy, context); /// Changes in columns may lead to changes in keys expression - metadata_copy.sorting_key = KeyDescription::getKeyFromAST(metadata_copy.sorting_key.definition_ast, metadata_copy.columns, context); + metadata_copy.sorting_key = KeyDescription::getKeyFromAST( + metadata_copy.sorting_key.definition_ast, + metadata_copy.columns, + context, + metadata_copy.sorting_key.additional_key_column); if (metadata_copy.primary_key.definition_ast != nullptr) { @@ -721,7 +725,7 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, const Context & co } else { - metadata_copy.primary_key = metadata_copy.sorting_key; + metadata_copy.primary_key = KeyDescription::getKeyFromAST(metadata_copy.sorting_key.definition_ast, metadata_copy.columns, context); metadata_copy.primary_key.definition_ast = nullptr; } diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 83c95d9a20f..58cbb2bb7d6 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -387,7 +387,7 @@ void IStorage::alter( } -void IStorage::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) +void IStorage::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const { for (const auto & command : commands) { diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index c06b969ebf5..1a5a0a0753d 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -359,7 +359,7 @@ public: /** Checks that alter commands can be applied to storage. For example, columns can be modified, * or primary key can be changes, etc. */ - virtual void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings); + virtual void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const; /** ALTER tables with regard to its partitions. * Should handle locks for each command on its own. diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index 5d44ced78db..a6e15bcf816 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -17,6 +17,7 @@ KeyDescription::KeyDescription(const KeyDescription & other) , sample_block(other.sample_block) , column_names(other.column_names) , data_types(other.data_types) + , additional_key_column(other.additional_key_column ? other.additional_key_column->clone() : nullptr) { } @@ -36,18 +37,23 @@ KeyDescription & KeyDescription::operator=(const KeyDescription & other) sample_block = other.sample_block; column_names = other.column_names; data_types = other.data_types; + if (other.additional_key_column) + additional_key_column = other.additional_key_column->clone(); + else + additional_key_column.reset(); return *this; } -KeyDescription KeyDescription::getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, ASTPtr additional_key_expression) +KeyDescription KeyDescription::getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, const ASTPtr & additional_key_column) { KeyDescription result; result.definition_ast = definition_ast; + result.additional_key_column = additional_key_column; result.expression_list_ast = extractKeyExpressionList(definition_ast); - if (additional_key_expression) - result.expression_list_ast->children.push_back(additional_key_expression); + if (additional_key_column != nullptr) + result.expression_list_ast->children.push_back(additional_key_column); const auto & children = result.expression_list_ast->children; for (const auto & child : children) diff --git a/src/Storages/KeyDescription.h b/src/Storages/KeyDescription.h index 980bd3baf00..c41154dec6f 100644 --- a/src/Storages/KeyDescription.h +++ b/src/Storages/KeyDescription.h @@ -30,9 +30,16 @@ struct KeyDescription /// Types from sample block ordered in columns order. DataTypes data_types; + /// Additional key column added by storage + ASTPtr additional_key_column; + /// Parse key structure from key definition. Requires all columns, available /// in storage. - static KeyDescription getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, ASTPtr additional_key_expression = nullptr); + static KeyDescription getKeyFromAST( + const ASTPtr & definition_ast, + const ColumnsDescription & columns, + const Context & context, + const ASTPtr & additional_key_expression = nullptr); KeyDescription() = default; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 4a65d05cc8e..9cd260b6748 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -144,7 +144,7 @@ MergeTreeData::MergeTreeData( setSettingsChanges(metadata_.settings_changes); const auto settings = getSettings(); - setProperties(metadata_, /*only_check*/ false, attach); + setProperties(metadata_, attach); /// NOTE: using the same columns list as is read when performing actual merges. merging_params.check(getColumns().getAllPhysical()); @@ -275,33 +275,13 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam } } -void MergeTreeData::setProperties(const StorageInMemoryMetadata & new_metadata, bool only_check, bool attach) +void MergeTreeData::checkProperties(const StorageInMemoryMetadata & new_metadata, bool attach) const { - KeyDescription new_primary_key = new_metadata.primary_key; - if (!new_metadata.sorting_key.definition_ast) throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS); - KeyDescription new_sorting_key; - if (merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) - new_sorting_key = KeyDescription::getKeyFromAST( - new_metadata.sorting_key.definition_ast, - new_metadata.columns, - global_context, - std::make_shared(merging_params.version_column)); - else - new_sorting_key = new_metadata.sorting_key; - - /// Primary key not defined at all - if (new_primary_key.definition_ast == nullptr) - { - /// We copy sorting key, and restore definition_ast to empty value, - /// because in merge tree code we chech, that our primary key is fake - /// (copied from sorting key, i.e. isPrimaryKeyDefined() == false, but - /// hasSortingKey() == true) - new_primary_key = new_metadata.sorting_key; - new_primary_key.definition_ast = nullptr; - } + KeyDescription new_sorting_key = new_metadata.sorting_key; + KeyDescription new_primary_key = new_metadata.primary_key; size_t sorting_key_size = new_sorting_key.column_names.size(); size_t primary_key_size = new_primary_key.column_names.size(); @@ -333,7 +313,7 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & new_metadata, auto all_columns = new_metadata.columns.getAllPhysical(); /// Order by check AST - if (hasSortingKey() && only_check) + if (hasSortingKey()) { /// This is ALTER, not CREATE/ATTACH TABLE. Let us check that all new columns used in the sorting key /// expression have just been added (so that the sorting order is guaranteed to be valid with the new key). @@ -400,15 +380,18 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & new_metadata, checkKeyExpression(*new_sorting_key.expression, new_sorting_key.sample_block, "Sorting"); - if (!only_check) - { - /// Other parts of metadata initialized is separate methods - setColumns(std::move(new_metadata.columns)); - setSecondaryIndices(std::move(new_metadata.secondary_indices)); - setConstraints(std::move(new_metadata.constraints)); - setSortingKey(std::move(new_sorting_key)); - setPrimaryKey(std::move(new_primary_key)); - } +} + +void MergeTreeData::setProperties(const StorageInMemoryMetadata & new_metadata, bool attach) +{ + checkProperties(new_metadata, attach); + + /// Other parts of metadata initialized is separate methods + setColumns(std::move(new_metadata.columns)); + setSecondaryIndices(std::move(new_metadata.secondary_indices)); + setConstraints(std::move(new_metadata.constraints)); + setSortingKey(std::move(new_metadata.sorting_key)); + setPrimaryKey(std::move(new_metadata.primary_key)); } namespace @@ -442,27 +425,6 @@ ExpressionActionsPtr MergeTreeData::getSortingKeyAndSkipIndicesExpression() cons return getCombinedIndicesExpression(getSortingKey(), getSecondaryIndices(), getColumns(), global_context); } -ASTPtr MergeTreeData::extractKeyExpressionList(const ASTPtr & node) -{ - if (!node) - return std::make_shared(); - - const auto * expr_func = node->as(); - - if (expr_func && expr_func->name == "tuple") - { - /// Primary key is specified in tuple, extract its arguments. - return expr_func->arguments->clone(); - } - else - { - /// Primary key consists of one column. - auto res = std::make_shared(); - res->children.push_back(node); - return res; - } -} - void MergeTreeData::initPartitionKey(const KeyDescription & new_partition_key) { @@ -521,8 +483,7 @@ void MergeTreeData::initPartitionKey(const KeyDescription & new_partition_key) } -/// Todo replace columns with TTL for columns -void MergeTreeData::setTTLExpressions(const StorageInMemoryMetadata & new_metadata, bool only_check) +void MergeTreeData::checkTTLExpressios(const StorageInMemoryMetadata & new_metadata) const { auto new_column_ttls = new_metadata.column_ttls_by_name; @@ -543,11 +504,7 @@ void MergeTreeData::setTTLExpressions(const StorageInMemoryMetadata & new_metada if (columns_ttl_forbidden.count(name)) throw Exception("Trying to set TTL for key column " + name, ErrorCodes::ILLEGAL_COLUMN); } - - if (!only_check) - setColumnTTLs(new_column_ttls); } - auto new_table_ttl = new_metadata.table_ttl; if (new_table_ttl.definition_ast) @@ -564,15 +521,18 @@ void MergeTreeData::setTTLExpressions(const StorageInMemoryMetadata & new_metada throw Exception(message, ErrorCodes::BAD_TTL_EXPRESSION); } } - - if (!only_check) - { - auto move_ttl_entries_lock = std::lock_guard(move_ttl_entries_mutex); - setTableTTLs(new_table_ttl); - } } } +/// Todo replace columns with TTL for columns +void MergeTreeData::setTTLExpressions(const StorageInMemoryMetadata & new_metadata) +{ + checkTTLExpressios(new_metadata); + setColumnTTLs(new_metadata.column_ttls_by_name); + auto move_ttl_entries_lock = std::lock_guard(move_ttl_entries_mutex); + setTableTTLs(new_metadata.table_ttl); +} + void MergeTreeData::checkStoragePolicy(const StoragePolicyPtr & new_storage_policy) const { @@ -1264,7 +1224,7 @@ bool isMetadataOnlyConversion(const IDataType * from, const IDataType * to) } -void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) +void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const { /// Check that needed transformations can be applied to the list of columns without considering type conversions. StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); @@ -1359,9 +1319,9 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S } } - setProperties(new_metadata, /* only_check = */ true); + checkProperties(new_metadata); - setTTLExpressions(new_metadata, /* only_check = */ true); + checkTTLExpressios(new_metadata); if (hasSettingsChanges()) { diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index a239f8cc0d7..521f80bed70 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -492,7 +492,7 @@ public: /// - all type conversions can be done. /// - columns corresponding to primary key, indices, sign, sampling expression and date are not affected. /// If something is wrong, throws an exception. - void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) override; + void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const override; /// Change MergeTreeSettings void changeSettings( @@ -508,12 +508,6 @@ public: broken_part_callback(name); } - /** Get the key expression AST as an ASTExpressionList. It can be specified - * in the tuple: (CounterID, Date), or as one column: CounterID. - */ - static ASTPtr extractKeyExpressionList(const ASTPtr & node); - - /// Check that the part is not broken and calculate the checksums for it if they are not present. MutableDataPartPtr loadPartAndFixMetadata(const VolumePtr & volume, const String & relative_path) const; @@ -780,11 +774,14 @@ protected: /// The same for clearOldTemporaryDirectories. std::mutex clear_old_temporary_directories_mutex; - void setProperties(const StorageInMemoryMetadata & new_metadata, bool only_check = false, bool attach = false); + void checkProperties(const StorageInMemoryMetadata & new_metadata, bool attach = false) const; + + void setProperties(const StorageInMemoryMetadata & new_metadata, bool attach = false); void initPartitionKey(const KeyDescription & new_partition_key); - void setTTLExpressions(const StorageInMemoryMetadata & new_metadata, bool only_check = false); + void checkTTLExpressios(const StorageInMemoryMetadata & new_metadata) const; + void setTTLExpressions(const StorageInMemoryMetadata & new_metadata); void checkStoragePolicy(const StoragePolicyPtr & new_storage_policy) const; diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 90eee680d65..050d0790bb5 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -417,6 +417,8 @@ static StoragePtr create(const StorageFactory::Arguments & args) ++arg_num; } + ASTPtr merging_param_key_arg = nullptr; + if (merging_params.mode == MergeTreeData::MergingParams::Collapsing) { if (!tryGetIdentifierNameInto(engine_args[arg_cnt - 1], merging_params.sign_column)) @@ -480,6 +482,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) ErrorCodes::BAD_ARGUMENTS); --arg_cnt; + merging_param_key_arg = std::make_shared(merging_params.version_column); } String date_column_name; @@ -502,13 +505,15 @@ static StoragePtr create(const StorageFactory::Arguments & args) "If you don't want this table to be sorted, use ORDER BY tuple()", ErrorCodes::BAD_ARGUMENTS); - metadata.sorting_key = KeyDescription::getKeyFromAST(args.storage_def->order_by->ptr(), metadata.columns, args.context); + metadata.sorting_key = KeyDescription::getKeyFromAST(args.storage_def->order_by->ptr(), metadata.columns, args.context, merging_param_key_arg); if (args.storage_def->primary_key) + { metadata.primary_key = KeyDescription::getKeyFromAST(args.storage_def->primary_key->ptr(), metadata.columns, args.context); + } else { - metadata.primary_key = metadata.sorting_key; + metadata.primary_key = KeyDescription::getKeyFromAST(args.storage_def->order_by->ptr(), metadata.columns, args.context); metadata.primary_key.definition_ast = nullptr; } @@ -560,7 +565,11 @@ static StoragePtr create(const StorageFactory::Arguments & args) } /// Now only two parameters remain - primary_key, index_granularity. - metadata.sorting_key = KeyDescription::getKeyFromAST(engine_args[arg_num], metadata.columns, args.context); + metadata.sorting_key = KeyDescription::getKeyFromAST(engine_args[arg_num], metadata.columns, args.context, merging_param_key_arg); + + metadata.primary_key = KeyDescription::getKeyFromAST(engine_args[arg_num], metadata.columns, args.context); + metadata.primary_key.definition_ast = nullptr; + ++arg_num; const auto * ast = engine_args[arg_num]->as(); diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 1f2c05bf7ad..08dc81b4945 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -723,7 +723,7 @@ void StorageBuffer::reschedule() flush_handle->scheduleAfter(std::min(min, max) * 1000); } -void StorageBuffer::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) +void StorageBuffer::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const { for (const auto & command : commands) { diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index 10a4482c801..02fd35136bf 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -85,7 +85,7 @@ public: bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context) const override; - void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override; + void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const override; /// The structure of the subordinate table is not checked and does not change. void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 7e7460a013f..3383c609520 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -538,7 +538,7 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const Context & c } -void StorageDistributed::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) +void StorageDistributed::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const { for (const auto & command : commands) { diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 4067012c449..63021e0a169 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -84,7 +84,7 @@ public: void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override; void renameOnDisk(const String & new_path_to_table_data); - void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override; + void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const override; /// in the sub-tables, you need to manually add and delete columns /// the structure of the sub-table is not checked diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 08984a6e1f3..a0c2fa87eb2 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -213,7 +213,7 @@ void StorageMaterializedView::alter( } -void StorageMaterializedView::checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) +void StorageMaterializedView::checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const { if (settings.allow_experimental_alter_materialized_view_structure) { diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 06bf659e05f..480c75aa114 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -41,7 +41,7 @@ public: void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; - void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) override; + void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const override; void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 64d0f11f853..8264eaa4cb6 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -372,7 +372,7 @@ DatabaseTablesIteratorPtr StorageMerge::getDatabaseIterator(const Context & cont } -void StorageMerge::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) +void StorageMerge::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const { for (const auto & command : commands) { diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 401c5facf0c..adf4a40e675 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -37,7 +37,7 @@ public: size_t max_block_size, unsigned num_streams) override; - void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override; + void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const override; /// you need to add and remove columns in the sub-tables manually /// the structure of sub-tables is not checked diff --git a/src/Storages/StorageNull.cpp b/src/Storages/StorageNull.cpp index b6e4605530d..182ce09ef96 100644 --- a/src/Storages/StorageNull.cpp +++ b/src/Storages/StorageNull.cpp @@ -32,7 +32,7 @@ void registerStorageNull(StorageFactory & factory) }); } -void StorageNull::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) +void StorageNull::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const { for (const auto & command : commands) { diff --git a/src/Storages/StorageNull.h b/src/Storages/StorageNull.h index d9aeb60c260..5fb4a16a24b 100644 --- a/src/Storages/StorageNull.h +++ b/src/Storages/StorageNull.h @@ -40,7 +40,7 @@ public: return std::make_shared(getSampleBlock()); } - void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override; + void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const override; void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 42adf4d2b45..7e42b75104d 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -492,7 +492,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column tuple->arguments->children = new_sorting_key_expr_list->children; order_by_ast = tuple; } - new_metadata.sorting_key = KeyDescription::getKeyFromAST(order_by_ast, new_metadata.columns, global_context); + new_metadata.sorting_key = KeyDescription::getKeyFromAST(order_by_ast, new_metadata.columns, global_context, new_metadata.sorting_key.additional_key_column); if (!isPrimaryKeyDefined()) { diff --git a/tests/queries/0_stateless/01213_alter_rename_primary_key_zookeeper.sql b/tests/queries/0_stateless/01213_alter_rename_primary_key_zookeeper.sql index 5c62d5d9107..daec7666e2d 100644 --- a/tests/queries/0_stateless/01213_alter_rename_primary_key_zookeeper.sql +++ b/tests/queries/0_stateless/01213_alter_rename_primary_key_zookeeper.sql @@ -17,11 +17,11 @@ INSERT INTO table_for_rename_pk SELECT toDate('2019-10-01') + number % 3, number SELECT key1, value1 FROM table_for_rename_pk WHERE key1 = 1 AND key2 = 1 AND key3 = 1; -ALTER TABLE table_for_rename_pk RENAME COLUMN key1 TO renamed_key1; --{serverError 524} +ALTER TABLE table_for_rename_pk RENAME COLUMN key1 TO renamed_key1; --{serverError 47} -ALTER TABLE table_for_rename_pk RENAME COLUMN key3 TO renamed_key3; --{serverError 524} +ALTER TABLE table_for_rename_pk RENAME COLUMN key3 TO renamed_key3; --{serverError 47} -ALTER TABLE table_for_rename_pk RENAME COLUMN key2 TO renamed_key2; --{serverError 524} +ALTER TABLE table_for_rename_pk RENAME COLUMN key2 TO renamed_key2; --{serverError 47} DROP TABLE IF EXISTS table_for_rename_pk NO DELAY; SELECT sleep(1) FORMAT Null; @@ -45,11 +45,11 @@ PRIMARY KEY (key1, key2); INSERT INTO table_for_rename_with_primary_key SELECT toDate('2019-10-01') + number % 3, number, number, number, toString(number), toString(number) from numbers(9); -ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key1 TO renamed_key1; --{serverError 524} +ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key1 TO renamed_key1; --{serverError 47} -ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key2 TO renamed_key2; --{serverError 524} +ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key2 TO renamed_key2; --{serverError 47} -ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key3 TO renamed_key3; --{serverError 524} +ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key3 TO renamed_key3; --{serverError 47} ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN value1 TO renamed_value1; --{serverError 524} From 24db834046e0dcb3631e67193853ddceb4206541 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Jun 2020 14:27:16 +0300 Subject: [PATCH 047/329] Style --- src/Storages/KeyDescription.cpp | 6 +++++- src/Storages/KeyDescription.h | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index a6e15bcf816..b1e74db2c58 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -45,7 +45,11 @@ KeyDescription & KeyDescription::operator=(const KeyDescription & other) } -KeyDescription KeyDescription::getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, const ASTPtr & additional_key_column) +KeyDescription KeyDescription::getKeyFromAST( + const ASTPtr & definition_ast, + const ColumnsDescription & columns, + const Context & context, + const ASTPtr & additional_key_column) { KeyDescription result; result.definition_ast = definition_ast; diff --git a/src/Storages/KeyDescription.h b/src/Storages/KeyDescription.h index c41154dec6f..97f48d435b2 100644 --- a/src/Storages/KeyDescription.h +++ b/src/Storages/KeyDescription.h @@ -30,7 +30,7 @@ struct KeyDescription /// Types from sample block ordered in columns order. DataTypes data_types; - /// Additional key column added by storage + /// Additional key column added by storage type ASTPtr additional_key_column; /// Parse key structure from key definition. Requires all columns, available @@ -39,7 +39,7 @@ struct KeyDescription const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, - const ASTPtr & additional_key_expression = nullptr); + const ASTPtr & additional_key_column = nullptr); KeyDescription() = default; From 9166ddea41812133d3ff1824c41cd36b6fa5a685 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 10 Jun 2020 15:53:12 +0300 Subject: [PATCH 048/329] fix segfault in show create table --- src/Databases/DatabaseMemory.cpp | 2 +- src/Interpreters/InterpreterShowCreateQuery.cpp | 2 +- .../01098_temporary_and_external_tables.sh | 11 ++++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 52b1f889943..cd559172197 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -68,7 +68,7 @@ ASTPtr DatabaseMemory::getCreateTableQueryImpl(const String & table_name, const { std::lock_guard lock{mutex}; auto it = create_queries.find(table_name); - if (it == create_queries.end()) + if (it == create_queries.end() || !it->second) { if (throw_on_error) throw Exception("There is no metadata of table " + table_name + " in database " + database_name, ErrorCodes::UNKNOWN_TABLE); diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 30005c7b169..b14baaafbb9 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -69,7 +69,7 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() create_query = DatabaseCatalog::instance().getDatabase(show_query->database)->getCreateDictionaryQuery(show_query->table); } - if (!create_query && show_query && show_query->temporary) + if (!create_query) throw Exception("Unable to show the create query of " + show_query->table + ". Maybe it was created by the system.", ErrorCodes::THERE_IS_NO_QUERY); if (!context.getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil) diff --git a/tests/queries/0_stateless/01098_temporary_and_external_tables.sh b/tests/queries/0_stateless/01098_temporary_and_external_tables.sh index c984f363c31..b671019ca35 100755 --- a/tests/queries/0_stateless/01098_temporary_and_external_tables.sh +++ b/tests/queries/0_stateless/01098_temporary_and_external_tables.sh @@ -3,7 +3,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -url="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/?session_id=test_01098" +url_without_session="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/?" +url="${url_without_session}session_id=test_01098" ${CLICKHOUSE_CURL} -m 30 -sSk "$url" --data "CREATE TEMPORARY TABLE tmp_table AS SELECT number AS n FROM numbers(42)" > /dev/null; @@ -14,3 +15,11 @@ echo "SELECT * FROM $full_tmp_name" | ${CLICKHOUSE_CURL} -m 60 -sSgk $url -d @- echo -ne '0\n1\n' | ${CLICKHOUSE_CURL} -m 30 -sSkF 'file=@-' "$url&file_format=CSV&file_types=UInt64&query=SELECT+sum((number+GLOBAL+IN+(SELECT+number+AS+n+FROM+remote('127.0.0.2',+numbers(5))+WHERE+n+GLOBAL+IN+(SELECT+*+FROM+tmp_table)+AND+n+GLOBAL+NOT+IN+(SELECT+*+FROM+file)+))+AS+res),+sum(number*res)+FROM+remote('127.0.0.2',+numbers(10))"; +echo -ne '0\n1\n' | ${CLICKHOUSE_CURL} -m 30 -sSkF 'file=@-' "$url&file_format=CSV&file_types=UInt64&query=SELECT+sleepEachRow(3)+FROM+file" > /dev/null & +sleep 1 +full_tmp_names=`echo "SELECT $name_expr FROM system.tables WHERE database='_temporary_and_external_tables' FORMAT TSV" | ${CLICKHOUSE_CURL} -m 30 -sSgk $url_without_session -d @-` +for name in $full_tmp_names +do + ${CLICKHOUSE_CURL} -m 30 -sSk "${url_without_session}query=SHOW+CREATE+TABLE+$name" 1>/dev/null 2>/dev/null +done; +wait From 08073903ed10aa15d1fbddea6d90929d7f9e4619 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 10 Jun 2020 17:24:42 +0300 Subject: [PATCH 049/329] fix query_id of http queries --- src/Interpreters/Context.cpp | 2 ++ .../0_stateless/01194_http_query_id.reference | 1 + tests/queries/0_stateless/01194_http_query_id.sh | 16 ++++++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 tests/queries/0_stateless/01194_http_query_id.reference create mode 100755 tests/queries/0_stateless/01194_http_query_id.sh diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index cbf00836103..7b636b84c68 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -166,6 +166,8 @@ public: if (!session.unique()) throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); + session->context.client_info = context.client_info; + return session; } diff --git a/tests/queries/0_stateless/01194_http_query_id.reference b/tests/queries/0_stateless/01194_http_query_id.reference new file mode 100644 index 00000000000..b8626c4cff2 --- /dev/null +++ b/tests/queries/0_stateless/01194_http_query_id.reference @@ -0,0 +1 @@ +4 diff --git a/tests/queries/0_stateless/01194_http_query_id.sh b/tests/queries/0_stateless/01194_http_query_id.sh new file mode 100755 index 00000000000..381ae67f88f --- /dev/null +++ b/tests/queries/0_stateless/01194_http_query_id.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +url="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?session_id=test_01194" +rnd=$RANDOM + +${CLICKHOUSE_CURL} -sS "$url&query=SELECT+$rnd,1" > /dev/null +${CLICKHOUSE_CURL} -sS "$url&query=SELECT+$rnd,2" > /dev/null +${CLICKHOUSE_CURL} -sS "$url" --data "SELECT $rnd,3" > /dev/null +${CLICKHOUSE_CURL} -sS "$url" --data "SELECT $rnd,4" > /dev/null + +${CLICKHOUSE_CURL} -sS "$url" --data "SYSTEM FLUSH LOGS" + +${CLICKHOUSE_CURL} -sS "$url&query=SELECT+count(DISTINCT+query_id)+FROM+system.query_log+WHERE+query+LIKE+'SELECT+$rnd%25'" From d4c8adbaffe0fea8adef8585bc91fa3524589b83 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 10 Jun 2020 18:06:37 +0300 Subject: [PATCH 050/329] Fix sanitizeBlock --- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 3010dfcfe12..ecfa011f1c8 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -113,7 +113,7 @@ bool sanitizeBlock(Block & block) return false; col.column = col.type->createColumn(); } - else if (isColumnConst(*col.column) && !col.column->empty()) + else if (!col.column->empty()) col.column = col.column->cloneEmpty(); } return true; From e2c6d090192d870edffc40e40deeb4c14417e5be Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 10 Jun 2020 18:06:55 +0300 Subject: [PATCH 051/329] Added test. --- .../01305_array_join_prewhere_in_subquery.reference | 1 + .../0_stateless/01305_array_join_prewhere_in_subquery.sql | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.reference create mode 100644 tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.sql diff --git a/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.reference b/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.sql b/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.sql new file mode 100644 index 00000000000..535dee5ebbe --- /dev/null +++ b/tests/queries/0_stateless/01305_array_join_prewhere_in_subquery.sql @@ -0,0 +1,5 @@ +drop table if exists h; +create table h (EventDate Date, CounterID UInt64, WatchID UInt64) engine = MergeTree order by (CounterID, EventDate); +insert into h values ('2020-06-10', 16671268, 1); +SELECT count() from h ARRAY JOIN [1] AS a PREWHERE WatchID IN (SELECT toUInt64(1)) WHERE (EventDate = '2020-06-10') AND (CounterID = 16671268); +drop table if exists h; From 8c1981757504b334f129e3aa1845ecf85d843768 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 10 Jun 2020 18:30:50 +0300 Subject: [PATCH 052/329] lock_guard --- src/Interpreters/SystemLog.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 1f889d53aec..9f5578614c0 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -435,7 +435,7 @@ void SystemLog::flushImpl(const std::vector & to_flush, template void SystemLog::prepareTable() { - std::unique_lock prepare_lock(prepare_mutex); + std::lock_guard prepare_lock(prepare_mutex); String description = table_id.getNameForLogs(); From bf262a3b04fae0023394b380c09f52937fe23217 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 10 Jun 2020 18:44:28 +0300 Subject: [PATCH 053/329] more lock_guard --- src/Interpreters/SystemLog.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 9f5578614c0..3c07af8c985 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -217,7 +217,7 @@ SystemLog::SystemLog(Context & context_, template void SystemLog::startup() { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); saving_thread = ThreadFromGlobalPool([this] { savingThreadFunction(); }); } @@ -231,7 +231,7 @@ void SystemLog::add(const LogElement & element) /// Otherwise the tests like 01017_uniqCombined_memory_usage.sql will be flacky. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (is_shutdown) return; @@ -310,7 +310,7 @@ template void SystemLog::stopFlushThread() { { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!saving_thread.joinable()) { @@ -423,7 +423,7 @@ void SystemLog::flushImpl(const std::vector & to_flush, } { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); flushed_before = to_flush_end; flush_event.notify_all(); } From e221a64c4b76b35eeb4f37945b4b4960479708b7 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Jun 2020 18:56:18 +0300 Subject: [PATCH 054/329] Better test --- .../integration/test_always_fetch_merged/test.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_always_fetch_merged/test.py b/tests/integration/test_always_fetch_merged/test.py index 63ab6f6b5ea..79458633081 100644 --- a/tests/integration/test_always_fetch_merged/test.py +++ b/tests/integration/test_always_fetch_merged/test.py @@ -55,9 +55,13 @@ def test_replica_always_download(started_cluster): node1.query("SYSTEM START MERGES") - time.sleep(3) - - node1_parts = node1.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1").strip() - node2_parts = node2.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1").strip() - assert int(node1_parts) < 10 # something merged - assert int(node2_parts) < 10 + for i in range(30): + node1_parts = node1.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1").strip() + node2_parts = node2.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1").strip() + if int(node1_parts) < 10 and int(node2_parts) < 10: + break + else: + time.sleep(0.5) + else: + assert int(node1_parts) < 10 + assert int(node2_parts) < 10 From 8418612e095d611bc5c214a29ab849239d386497 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 10 Jun 2020 20:56:47 +0300 Subject: [PATCH 055/329] Add unbundled boost support Follow-up-for: #11390 Cc: @@abyss7 --- contrib/boost-cmake/CMakeLists.txt | 33 +++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index fb7b236d30d..4360304403f 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -97,5 +97,36 @@ if (USE_INTERNAL_BOOST_LIBRARY) add_library (boost::system ALIAS _boost_system) target_include_directories (_boost_system PRIVATE ${LIBRARY_DIR}) else () - message (FATAL_ERROR "TODO: external Boost library is not supported!") + # 1.70 like in contrib/boost + set(BOOST_VERSION 1.70) + + find_package(Boost ${BOOST_VERSION} COMPONENTS + system + filesystem + iostreams + program_options + regex + REQUIRED) + + add_library (_boost_headers_only INTERFACE) + add_library (boost::headers_only ALIAS _boost_headers_only) + target_include_directories (_boost_headers_only SYSTEM BEFORE INTERFACE ${Boost_INCLUDE_DIR}) + + add_library (_boost_filesystem INTERFACE) + add_library (_boost_iostreams INTERFACE) + add_library (_boost_program_options INTERFACE) + add_library (_boost_regex INTERFACE) + add_library (_boost_system INTERFACE) + + target_link_libraries (_boost_filesystem INTERFACE ${Boost_FILESYSTEM_LIBRARY}) + target_link_libraries (_boost_iostreams INTERFACE ${Boost_IOSTREAMS_LIBRARY}) + target_link_libraries (_boost_program_options INTERFACE ${Boost_PROGRAM_OPTIONS_LIBRARY}) + target_link_libraries (_boost_regex INTERFACE ${Boost_REGEX_LIBRARY}) + target_link_libraries (_boost_system INTERFACE ${Boost_SYSTEM_LIBRARY}) + + add_library (boost::filesystem ALIAS _boost_filesystem) + add_library (boost::iostreams ALIAS _boost_iostreams) + add_library (boost::program_options ALIAS _boost_program_options) + add_library (boost::regex ALIAS _boost_regex) + add_library (boost::system ALIAS _boost_system) endif () From 1f3c640da2c08ea98576e6641a9a53d66902f4fa Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 10 Jun 2020 20:56:47 +0300 Subject: [PATCH 056/329] Use unbundled boost for unbundled builds --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 85dd3cc421c..ccb01a4df92 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -142,7 +142,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if unbundled: # TODO: fix build with ENABLE_RDKAFKA - cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0 -DUSE_INTERNAL_BOOST_LIBRARY=1') + cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0') if split_binary: cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') From dd3cf0fe0aaf528a555d51487bc5c74c83d31591 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Jun 2020 21:02:45 +0300 Subject: [PATCH 057/329] Don't miss columns TTLs update --- src/Storages/StorageReplicatedMergeTree.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 7e42b75104d..1465cef58e1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -474,8 +474,17 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column { StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); if (new_columns != new_metadata.columns) + { new_metadata.columns = new_columns; + new_metadata.column_ttls_by_name.clear(); + for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs()) + { + auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, global_context, new_metadata.primary_key); + new_metadata.column_ttls_by_name[name] = new_ttl_entry; + } + } + if (!metadata_diff.empty()) { if (metadata_diff.sorting_key_changed) From 9c86c128915bf5ae350273a09e6f114bac4325bd Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 10 Jun 2020 21:11:30 +0300 Subject: [PATCH 058/329] Update ParserShowTablesQuery.cpp --- src/Parsers/ParserShowTablesQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ParserShowTablesQuery.cpp b/src/Parsers/ParserShowTablesQuery.cpp index c60e442542d..ee50d23ffc8 100644 --- a/src/Parsers/ParserShowTablesQuery.cpp +++ b/src/Parsers/ParserShowTablesQuery.cpp @@ -73,7 +73,7 @@ bool ParserShowTablesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec String cluster_str; if (!parseIdentifierOrStringLiteral(pos, expected, cluster_str)) return false; - + query->cluster_str = std::move(cluster_str); } else From 982c85ac6c61264560684b433ed5104fdd99c0c3 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 10 Jun 2020 21:13:16 +0300 Subject: [PATCH 059/329] Update 01293_show_clusters.reference --- tests/queries/0_stateless/01293_show_clusters.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01293_show_clusters.reference b/tests/queries/0_stateless/01293_show_clusters.reference index 85a14155529..39e25143131 100644 --- a/tests/queries/0_stateless/01293_show_clusters.reference +++ b/tests/queries/0_stateless/01293_show_clusters.reference @@ -1,7 +1,7 @@ test_cluster_two_shards +test_cluster_two_shards_different_databases test_cluster_two_shards_localhost test_shard_localhost -test_shard_localhost[1] test_shard_localhost_secure test_unavailable_shard test_cluster_two_shards From 3d9424fd5a26aabfbc863984a6e2364eb21e1110 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 21:54:55 +0300 Subject: [PATCH 060/329] perl -pi -e 's/repo\.yandex\.ru\/clickhouse/repo.clickhouse.tech/g' {} --- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- docs/en/getting-started/install.md | 2 +- docs/es/getting-started/install.md | 2 +- docs/fa/getting-started/install.md | 2 +- docs/fr/getting-started/install.md | 2 +- docs/ja/getting-started/install.md | 2 +- docs/ru/getting-started/install.md | 16 ++++++++-------- docs/tr/getting-started/install.md | 2 +- docs/zh/getting-started/install.md | 6 +++--- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 5ca7e508d56..453c165afad 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:18.04 -ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" +ARG repository="deb http://repo.clickhouse.tech/deb/stable/ main/" ARG version=20.5.1.* RUN apt-get update \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 93f192c3f3c..45565ec659e 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:18.04 -ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" +ARG repository="deb http://repo.clickhouse.tech/deb/stable/ main/" ARG version=20.5.1.* ARG gosu_ver=1.10 diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 1dd756ed7c2..f215e21288e 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:18.04 -ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" +ARG repository="deb http://repo.clickhouse.tech/deb/stable/ main/" ARG version=20.5.1.* RUN apt-get update && \ diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index e917b8ef58c..7c8ae631e1a 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -64,7 +64,7 @@ You can also download and install packages manually from [here](https://repo.cli It is recommended to use official pre-compiled `tgz` archives for all Linux distributions, where installation of `deb` or `rpm` packages is not possible. -The required version can be downloaded with `curl` or `wget` from repository https://repo.yandex.ru/clickhouse/tgz/. +The required version can be downloaded with `curl` or `wget` from repository https://repo.clickhouse.tech/tgz/. After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest version: ``` bash diff --git a/docs/es/getting-started/install.md b/docs/es/getting-started/install.md index 83f5fff8af2..89b5735f192 100644 --- a/docs/es/getting-started/install.md +++ b/docs/es/getting-started/install.md @@ -66,7 +66,7 @@ También puede descargar e instalar paquetes manualmente desde [aqui](https://re Se recomienda utilizar pre-compilado oficial `tgz` para todas las distribuciones de Linux, donde la instalación de `deb` o `rpm` paquetes no es posible. -La versión requerida se puede descargar con `curl` o `wget` desde el repositorio https://repo.yandex.ru/clickhouse/tgz/. +La versión requerida se puede descargar con `curl` o `wget` desde el repositorio https://repo.clickhouse.tech/tgz/. Después de eso, los archivos descargados deben desempaquetarse e instalarse con scripts de instalación. Ejemplo para la última versión: ``` bash diff --git a/docs/fa/getting-started/install.md b/docs/fa/getting-started/install.md index ec0b67944d9..d3e39e97c80 100644 --- a/docs/fa/getting-started/install.md +++ b/docs/fa/getting-started/install.md @@ -67,7 +67,7 @@ sudo yum install clickhouse-server clickhouse-client توصیه می شود به استفاده از رسمی از پیش وارد شده `tgz` بایگانی برای همه توزیع های لینوکس, که نصب و راه اندازی `deb` یا `rpm` بسته امکان پذیر نیست. -نسخه مورد نیاز را می توان با دانلود `curl` یا `wget` از مخزن https://repo.yandex.ru/clickhouse/tgz/. +نسخه مورد نیاز را می توان با دانلود `curl` یا `wget` از مخزن https://repo.clickhouse.tech/tgz/. پس از که دانلود بایگانی باید غیر بستهای و نصب شده با اسکریپت نصب و راه اندازی. به عنوان مثال برای جدیدترین نسخه: ``` bash diff --git a/docs/fr/getting-started/install.md b/docs/fr/getting-started/install.md index 770c4cf8e42..3659369fb4a 100644 --- a/docs/fr/getting-started/install.md +++ b/docs/fr/getting-started/install.md @@ -66,7 +66,7 @@ Vous pouvez également télécharger et installer des paquets manuellement à pa Il est recommandé d'utiliser officiel pré-compilé `tgz` archives pour toutes les distributions Linux, où l'installation de `deb` ou `rpm` les emballages n'est pas possible. -La version requise peut être téléchargée avec `curl` ou `wget` depuis le référentiel https://repo.yandex.ru/clickhouse/tgz/. +La version requise peut être téléchargée avec `curl` ou `wget` depuis le référentiel https://repo.clickhouse.tech/tgz/. Après cela, les archives téléchargées doivent être décompressées et installées avec des scripts d'installation. Exemple pour la dernière version: ``` bash diff --git a/docs/ja/getting-started/install.md b/docs/ja/getting-started/install.md index 9710fdae6a7..66867652811 100644 --- a/docs/ja/getting-started/install.md +++ b/docs/ja/getting-started/install.md @@ -66,7 +66,7 @@ sudo yum install clickhouse-server clickhouse-client 公式の事前コンパイルを使用することをお勧めします `tgz` のインストール `deb` または `rpm` パッケージはできません。 -必要なバージョンは次のとおりです `curl` または `wget` リポジトリからhttps://repo.yandex.ru/clickhouse/tgz/. +必要なバージョンは次のとおりです `curl` または `wget` リポジトリからhttps://repo.clickhouse.tech/tgz/. その後、アーカイブをダウンロードは開梱と設置と設置のためのイントロダクションです。 最新バージョンの例: ``` bash diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index 04712328844..19943f182d8 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -34,8 +34,8 @@ $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not ``` bash sudo yum install yum-utils -sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +sudo rpm --import https://repo.clickhouse.tech/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.clickhouse.tech/rpm/stable/x86_64 ``` Для использования наиболее свежих версий нужно заменить `stable` на `testing` (рекомендуется для тестовых окружений). Также иногда доступен `prestable`. @@ -46,21 +46,21 @@ sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/ sudo yum install clickhouse-server clickhouse-client ``` -Также есть возможность установить пакеты вручную, скачав отсюда: https://repo.yandex.ru/clickhouse/rpm/stable/x86\_64. +Также есть возможность установить пакеты вручную, скачав отсюда: https://repo.clickhouse.tech/rpm/stable/x86\_64. ### Из Tgz архивов {#from-tgz-archives} Команда ClickHouse в Яндексе рекомендует использовать предкомпилированные бинарники из `tgz` архивов для всех дистрибутивов, где невозможна установка `deb` и `rpm` пакетов. -Интересующую версию архивов можно скачать вручную с помощью `curl` или `wget` из репозитория https://repo.yandex.ru/clickhouse/tgz/. +Интересующую версию архивов можно скачать вручную с помощью `curl` или `wget` из репозитория https://repo.clickhouse.tech/tgz/. После этого архивы нужно распаковать и воспользоваться скриптами установки. Пример установки самой свежей версии: ``` bash export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` -curl -O https://repo.yandex.ru/clickhouse/tgz/clickhouse-common-static-$LATEST_VERSION.tgz -curl -O https://repo.yandex.ru/clickhouse/tgz/clickhouse-common-static-dbg-$LATEST_VERSION.tgz -curl -O https://repo.yandex.ru/clickhouse/tgz/clickhouse-server-$LATEST_VERSION.tgz -curl -O https://repo.yandex.ru/clickhouse/tgz/clickhouse-client-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.tech/tgz/clickhouse-common-static-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.tech/tgz/clickhouse-common-static-dbg-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.tech/tgz/clickhouse-server-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.tech/tgz/clickhouse-client-$LATEST_VERSION.tgz tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh diff --git a/docs/tr/getting-started/install.md b/docs/tr/getting-started/install.md index 3bf319430bd..361f59b3a10 100644 --- a/docs/tr/getting-started/install.md +++ b/docs/tr/getting-started/install.md @@ -66,7 +66,7 @@ Ayrıca paketleri manuel olarak indirebilir ve yükleyebilirsiniz [burada](https Resmi önceden derlenmiş kullanılması tavsiye edilir `tgz` Arch ,iv ,es for tüm Linux dağıtım installationları, kurulumu `deb` veya `rpm` paketler mümkün değildir. -Gerekli sürümü ile indirilebilir `curl` veya `wget` depo fromdan https://repo.yandex.ru/clickhouse/tgz/. +Gerekli sürümü ile indirilebilir `curl` veya `wget` depo fromdan https://repo.clickhouse.tech/tgz/. Bundan sonra indirilen arşivler açılmalı ve kurulum komut dosyaları ile kurulmalıdır. En son sürüm için örnek: ``` bash diff --git a/docs/zh/getting-started/install.md b/docs/zh/getting-started/install.md index 32eb7fa0a82..bbc79c3bf78 100644 --- a/docs/zh/getting-started/install.md +++ b/docs/zh/getting-started/install.md @@ -34,8 +34,8 @@ Yandex ClickHouse团队建议使用官方预编译的`rpm`软件包，用于Cent ``` bash sudo yum install yum-utils -sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +sudo rpm --import https://repo.clickhouse.tech/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.clickhouse.tech/rpm/stable/x86_64 ``` 如果您想使用最新版本，请将`stable`替换为`testing`（建议您在测试环境中使用）。 @@ -46,7 +46,7 @@ sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/ sudo yum install clickhouse-server clickhouse-client ``` -您也可以从此处手动下载和安装软件包：https://repo.yandex.ru/clickhouse/rpm/stable/x86_64。 +您也可以从此处手动下载和安装软件包：https://repo.clickhouse.tech/rpm/stable/x86_64。 ### 来自Docker {#from-docker-image} From e4be52f35f8e019c3e9fb37100fae7850858579e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 10 Jun 2020 22:17:30 +0300 Subject: [PATCH 061/329] Add system.asynchronous_metric_log --- docs/en/operations/system-tables.md | 4 ++ programs/server/config.xml | 14 ++++ src/Interpreters/AsynchronousMetrics.cpp | 77 +++++++++++---------- src/Interpreters/AsynchronousMetrics.h | 17 ++--- src/Interpreters/Context.cpp | 11 +++ src/Interpreters/Context.h | 2 + src/Interpreters/InterpreterSystemQuery.cpp | 4 +- src/Interpreters/SystemLog.cpp | 7 ++ src/Interpreters/SystemLog.h | 3 + src/Interpreters/ya.make | 1 + 10 files changed, 93 insertions(+), 47 deletions(-) diff --git a/docs/en/operations/system-tables.md b/docs/en/operations/system-tables.md index 7b76f737824..28f448b632c 100644 --- a/docs/en/operations/system-tables.md +++ b/docs/en/operations/system-tables.md @@ -83,6 +83,10 @@ SELECT * FROM system.asynchronous_metrics LIMIT 10 - [system.events](#system_tables-events) — Contains a number of events that have occurred. - [system.metric\_log](#system_tables-metric_log) — Contains a history of metrics values from tables `system.metrics` и `system.events`. +## system.asynchronous_metric_log {#system-tables-async-log} + +Contains the historical values for `system.asynchronous_log` (see [system.asynchronous_metrics](#system_tables-asynchronous_metrics)) + ## system.clusters {#system-clusters} Contains information about clusters available in the config file and the servers in them. diff --git a/programs/server/config.xml b/programs/server/config.xml index ba870d8a8ea..944181ceee4 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -481,6 +481,20 @@ 1000 + + + system + asynchronous_metric_log
+ + 300000 + + diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 09622302893..6cd8fafa2a8 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -37,7 +38,7 @@ AsynchronousMetrics::~AsynchronousMetrics() try { { - std::lock_guard lock{wait_mutex}; + std::lock_guard lock{mutex}; quit = true; } @@ -51,17 +52,10 @@ AsynchronousMetrics::~AsynchronousMetrics() } -AsynchronousMetrics::Container AsynchronousMetrics::getValues() const +AsynchronousMetricValues AsynchronousMetrics::getValues() const { - std::lock_guard lock{container_mutex}; - return container; -} - - -void AsynchronousMetrics::set(const std::string & name, Value value) -{ - std::lock_guard lock{container_mutex}; - container[name] = value; + std::lock_guard lock{mutex}; + return values; } @@ -69,8 +63,6 @@ void AsynchronousMetrics::run() { setThreadName("AsyncMetrics"); - std::unique_lock lock{wait_mutex}; - /// Next minute + 30 seconds. To be distant with moment of transmission of metrics, see MetricsTransmitter. const auto get_next_minute = [] { @@ -89,6 +81,7 @@ void AsynchronousMetrics::run() tryLogCurrentException(__PRETTY_FUNCTION__); } + std::unique_lock lock{mutex}; if (wait_cond.wait_until(lock, get_next_minute(), [this] { return quit; })) break; } @@ -113,41 +106,43 @@ static void calculateMaxAndSum(Max & max, Sum & sum, T x) void AsynchronousMetrics::update() { + AsynchronousMetricValues new_values; + { if (auto mark_cache = context.getMarkCache()) { - set("MarkCacheBytes", mark_cache->weight()); - set("MarkCacheFiles", mark_cache->count()); + new_values["MarkCacheBytes"] = mark_cache->weight(); + new_values["MarkCacheFiles"] = mark_cache->count(); } } { if (auto uncompressed_cache = context.getUncompressedCache()) { - set("UncompressedCacheBytes", uncompressed_cache->weight()); - set("UncompressedCacheCells", uncompressed_cache->count()); + new_values["UncompressedCacheBytes"] = uncompressed_cache->weight(); + new_values["UncompressedCacheCells"] = uncompressed_cache->count(); } } #if USE_EMBEDDED_COMPILER { if (auto compiled_expression_cache = context.getCompiledExpressionCache()) - set("CompiledExpressionCacheCount", compiled_expression_cache->count()); + new_values["CompiledExpressionCacheCount"] = compiled_expression_cache->count(); } #endif - set("Uptime", context.getUptimeSeconds()); + new_values["Uptime"] = context.getUptimeSeconds(); /// Process memory usage according to OS #if defined(OS_LINUX) { MemoryStatisticsOS::Data data = memory_stat.get(); - set("MemoryVirtual", data.virt); - set("MemoryResident", data.resident); - set("MemoryShared", data.shared); - set("MemoryCode", data.code); - set("MemoryDataAndStack", data.data_and_stack); + new_values["MemoryVirtual"] = data.virt; + new_values["MemoryResident"] = data.resident; + new_values["MemoryShared"] = data.shared; + new_values["MemoryCode"] = data.code; + new_values["MemoryDataAndStack"] = data.data_and_stack; /// We must update the value of total_memory_tracker periodically. /// Otherwise it might be calculated incorrectly - it can include a "drift" of memory amount. @@ -228,21 +223,21 @@ void AsynchronousMetrics::update() } } - set("ReplicasMaxQueueSize", max_queue_size); - set("ReplicasMaxInsertsInQueue", max_inserts_in_queue); - set("ReplicasMaxMergesInQueue", max_merges_in_queue); + new_values["ReplicasMaxQueueSize"] = max_queue_size; + new_values["ReplicasMaxInsertsInQueue"] = max_inserts_in_queue; + new_values["ReplicasMaxMergesInQueue"] = max_merges_in_queue; - set("ReplicasSumQueueSize", sum_queue_size); - set("ReplicasSumInsertsInQueue", sum_inserts_in_queue); - set("ReplicasSumMergesInQueue", sum_merges_in_queue); + new_values["ReplicasSumQueueSize"] = sum_queue_size; + new_values["ReplicasSumInsertsInQueue"] = sum_inserts_in_queue; + new_values["ReplicasSumMergesInQueue"] = sum_merges_in_queue; - set("ReplicasMaxAbsoluteDelay", max_absolute_delay); - set("ReplicasMaxRelativeDelay", max_relative_delay); + new_values["ReplicasMaxAbsoluteDelay"] = max_absolute_delay; + new_values["ReplicasMaxRelativeDelay"] = max_relative_delay; - set("MaxPartCountForPartition", max_part_count_for_partition); + new_values["MaxPartCountForPartition"] = max_part_count_for_partition; - set("NumberOfDatabases", number_of_databases); - set("NumberOfTables", total_number_of_tables); + new_values["NumberOfDatabases"] = number_of_databases; + new_values["NumberOfTables"] = total_number_of_tables; } #if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 4 @@ -265,7 +260,7 @@ void AsynchronousMetrics::update() TYPE value{}; \ size_t size = sizeof(value); \ mallctl("stats." NAME, &value, &size, nullptr, 0); \ - set("jemalloc." NAME, value); \ + new_values["jemalloc." NAME] = value; \ } while (false); FOR_EACH_METRIC(GET_METRIC) @@ -276,6 +271,16 @@ void AsynchronousMetrics::update() #endif /// Add more metrics as you wish. + + // Log the new metrics. + if (auto log = context.getAsynchronousMetricLog()) + { + log->addValues(new_values); + } + + // Finally, update the current metrics. + std::lock_guard lock(mutex); + values = new_values; } } diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index ce6c0aae552..6817f545c8f 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -14,6 +14,9 @@ namespace DB class Context; +typedef double AsynchronousMetricValue; +typedef std::unordered_map AsynchronousMetricValues; + /** Periodically (each minute, starting at 30 seconds offset) * calculates and updates some metrics, @@ -29,21 +32,17 @@ public: ~AsynchronousMetrics(); - using Value = double; - using Container = std::unordered_map; /// Returns copy of all values. - Container getValues() const; + AsynchronousMetricValues getValues() const; private: Context & context; - bool quit {false}; - std::mutex wait_mutex; + mutable std::mutex mutex; std::condition_variable wait_cond; - - Container container; - mutable std::mutex container_mutex; + bool quit {false}; + AsynchronousMetricValues values; #if defined(OS_LINUX) MemoryStatisticsOS memory_stat; @@ -53,8 +52,6 @@ private: void run(); void update(); - - void set(const std::string & name, Value value); }; } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index cbf00836103..4a2408d64f1 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1676,6 +1676,17 @@ std::shared_ptr Context::getMetricLog() } +std::shared_ptr Context::getAsynchronousMetricLog() +{ + auto lock = getLock(); + + if (!shared->system_logs) + return {}; + + return shared->system_logs->asynchronous_metric_log; +} + + CompressionCodecPtr Context::chooseCompressionCodec(size_t part_size, double part_size_ratio) const { auto lock = getLock(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 1d46049fb92..5a4e959229f 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -80,6 +80,7 @@ class PartLog; class TextLog; class TraceLog; class MetricLog; +class AsynchronousMetricLog; struct MergeTreeSettings; class StorageS3Settings; class IDatabase; @@ -526,6 +527,7 @@ public: std::shared_ptr getTraceLog(); std::shared_ptr getTextLog(); std::shared_ptr getMetricLog(); + std::shared_ptr getAsynchronousMetricLog(); /// Returns an object used to log opertaions with parts if it possible. /// Provide table name to make required cheks. diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 9ebdb155643..f35bafbe25a 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -306,7 +307,8 @@ BlockIO InterpreterSystemQuery::execute() [&] () { if (auto query_thread_log = context.getQueryThreadLog()) query_thread_log->flush(); }, [&] () { if (auto trace_log = context.getTraceLog()) trace_log->flush(); }, [&] () { if (auto text_log = context.getTextLog()) text_log->flush(); }, - [&] () { if (auto metric_log = context.getMetricLog()) metric_log->flush(); } + [&] () { if (auto metric_log = context.getMetricLog()) metric_log->flush(); }, + [&] () { if (auto asynchronous_metric_log = context.getAsynchronousMetricLog()) asynchronous_metric_log->flush(); } ); break; case Type::STOP_LISTEN_QUERIES: diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index 9ce2f7a4d0e..d79edde7052 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -75,6 +76,9 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi trace_log = createSystemLog(global_context, "system", "trace_log", config, "trace_log"); text_log = createSystemLog(global_context, "system", "text_log", config, "text_log"); metric_log = createSystemLog(global_context, "system", "metric_log", config, "metric_log"); + asynchronous_metric_log = createSystemLog( + global_context, "system", "asynchronous_metric_log", config, + "asynchronous_metric_log"); if (query_log) logs.emplace_back(query_log.get()); @@ -88,6 +92,9 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi logs.emplace_back(text_log.get()); if (metric_log) logs.emplace_back(metric_log.get()); + if (asynchronous_metric_log) + logs.emplace_back(asynchronous_metric_log.get()); + try { diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index dd2f815ce92..e49ce574478 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -69,6 +69,7 @@ class PartLog; class TextLog; class TraceLog; class MetricLog; +class AsynchronousMetricLog; class ISystemLog @@ -99,6 +100,8 @@ struct SystemLogs std::shared_ptr trace_log; /// Used to log traces from query profiler std::shared_ptr text_log; /// Used to log all text messages. std::shared_ptr metric_log; /// Used to log all metrics. + /// Metrics from system.asynchronous_metrics. + std::shared_ptr asynchronous_metric_log; std::vector logs; }; diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 178c3ee3125..29be5d3c216 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -105,6 +105,7 @@ SRCS( MarkTableIdentifiersVisitor.cpp MergeJoin.cpp MetricLog.cpp + AsynchronousMetricLog.cpp MutationsInterpreter.cpp NullableUtils.cpp OptimizeIfChains.cpp From a07b296ee4afa8c9d29dceb9af3b3203d7f78e7b Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 22:26:25 +0300 Subject: [PATCH 062/329] Update Dockerfile --- docker/client/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 453c165afad..493cdaac543 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:18.04 -ARG repository="deb http://repo.clickhouse.tech/deb/stable/ main/" +ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" ARG version=20.5.1.* RUN apt-get update \ From f6b5cd7de212fede834389e6f70a24619bf89203 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 22:26:44 +0300 Subject: [PATCH 063/329] Update Dockerfile --- docker/server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 45565ec659e..3a16a1fd158 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:18.04 -ARG repository="deb http://repo.clickhouse.tech/deb/stable/ main/" +ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" ARG version=20.5.1.* ARG gosu_ver=1.10 From 7b54ff02f05f2a4ee93e0f467774ea9a2a533229 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 10 Jun 2020 22:27:05 +0300 Subject: [PATCH 064/329] Collect async metric log in perf test --- docker/test/performance-comparison/compare.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index f7986689020..a2760907cb3 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -198,12 +198,14 @@ function get_profiles clickhouse-client --port 9001 --query "select * from system.trace_log format TSVWithNamesAndTypes" > left-trace-log.tsv ||: & clickhouse-client --port 9001 --query "select arrayJoin(trace) addr, concat(splitByChar('/', addressToLine(addr))[-1], '#', demangle(addressToSymbol(addr)) ) name from system.trace_log group by addr format TSVWithNamesAndTypes" > left-addresses.tsv ||: & clickhouse-client --port 9001 --query "select * from system.metric_log format TSVWithNamesAndTypes" > left-metric-log.tsv ||: & + clickhouse-client --port 9001 --query "select * from system.asynchronous_metric_log format TSVWithNamesAndTypes" > left-async-metric-log.tsv ||: & clickhouse-client --port 9002 --query "select * from system.query_log where type = 2 format TSVWithNamesAndTypes" > right-query-log.tsv ||: & clickhouse-client --port 9002 --query "select * from system.query_thread_log format TSVWithNamesAndTypes" > right-query-thread-log.tsv ||: & clickhouse-client --port 9002 --query "select * from system.trace_log format TSVWithNamesAndTypes" > right-trace-log.tsv ||: & clickhouse-client --port 9002 --query "select arrayJoin(trace) addr, concat(splitByChar('/', addressToLine(addr))[-1], '#', demangle(addressToSymbol(addr)) ) name from system.trace_log group by addr format TSVWithNamesAndTypes" > right-addresses.tsv ||: & clickhouse-client --port 9002 --query "select * from system.metric_log format TSVWithNamesAndTypes" > right-metric-log.tsv ||: & + clickhouse-client --port 9002 --query "select * from system.asynchronous_metric_log format TSVWithNamesAndTypes" > right-async-metric-log.tsv ||: & wait From e5cd2716da4fc25cf1200410ea4e64152333eecf Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 22:27:05 +0300 Subject: [PATCH 065/329] Update Dockerfile --- docker/test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index f215e21288e..6673d32c2e2 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:18.04 -ARG repository="deb http://repo.clickhouse.tech/deb/stable/ main/" +ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" ARG version=20.5.1.* RUN apt-get update && \ From fd3ff19868c94270a33d6fde51f7429597c7d8bd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 10 Jun 2020 22:34:23 +0300 Subject: [PATCH 066/329] Fix trivial error in log message #11399 --- programs/server/Server.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8b58c5664b6..9734bafe30e 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -604,7 +604,8 @@ int Server::main(const std::vector & /*args*/) if (uncompressed_cache_size > max_cache_size) { uncompressed_cache_size = max_cache_size; - LOG_INFO(log, "Uncompressed cache size was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(uncompressed_cache_size)); + LOG_INFO(log, "Uncompressed cache size was lowered to {} because the system has low amount of memory", + formatReadableSizeWithBinarySuffix(uncompressed_cache_size)); } global_context->setUncompressedCache(uncompressed_cache_size); @@ -619,7 +620,8 @@ int Server::main(const std::vector & /*args*/) if (mark_cache_size > max_cache_size) { mark_cache_size = max_cache_size; - LOG_INFO(log, "Mark cache size was lowered to {} because the system has low amount of memory", formatReadableSizeWithBinarySuffix(uncompressed_cache_size)); + LOG_INFO(log, "Mark cache size was lowered to {} because the system has low amount of memory", + formatReadableSizeWithBinarySuffix(mark_cache_size)); } global_context->setMarkCache(mark_cache_size); From b5b3cb1c05300cca486c159a5fb034504e862496 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 23:48:54 +0300 Subject: [PATCH 067/329] Update index.md --- docs/en/sql-reference/statements/select/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 8224bf1e798..565b5fbcd5d 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -1,12 +1,12 @@ --- toc_priority: 33 toc_folder_title: SELECT -toc_title: Queries Syntax +toc_title: Query Syntax --- -# SELECT Queries Syntax {#select-queries-syntax} +# SELECT Query {#select-queries-syntax} -`SELECT` performs data retrieval. +`SELECT` queries perform data retrieval. By default, the requested data is returned to the client, while in conjunction with [INSERT INTO](../insert-into.md) it can be forwarded to a different table. ``` sql [WITH expr_list|(subquery)] From f433ffaec645bae06ae654dddd65efaf17b6513f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 23:49:37 +0300 Subject: [PATCH 068/329] Update index.md --- docs/en/sql-reference/statements/select/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 565b5fbcd5d..87dde4839ff 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -2,6 +2,7 @@ toc_priority: 33 toc_folder_title: SELECT toc_title: Query Syntax +title: SELECT Query --- # SELECT Query {#select-queries-syntax} From 8ec3ab9c9fef665f0159bc7aab206e3e3791f61b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 10 Jun 2020 23:57:54 +0300 Subject: [PATCH 069/329] Require less recent boost - 1.67 --- contrib/boost-cmake/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index 4360304403f..e92fe4b7159 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -98,7 +98,8 @@ if (USE_INTERNAL_BOOST_LIBRARY) target_include_directories (_boost_system PRIVATE ${LIBRARY_DIR}) else () # 1.70 like in contrib/boost - set(BOOST_VERSION 1.70) + # 1.67 on CI + set(BOOST_VERSION 1.67) find_package(Boost ${BOOST_VERSION} COMPONENTS system From a6648516f45307949c3fe82ee2fe35aab4f841ef Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 11 Jun 2020 00:16:58 +0300 Subject: [PATCH 070/329] Add files; flush more often --- programs/server/config.xml | 4 +- src/Interpreters/AsynchronousMetricLog.cpp | 64 ++++++++++++++++++++++ src/Interpreters/AsynchronousMetricLog.h | 41 ++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 src/Interpreters/AsynchronousMetricLog.cpp create mode 100644 src/Interpreters/AsynchronousMetricLog.h diff --git a/programs/server/config.xml b/programs/server/config.xml index 944181ceee4..0ceba85593a 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -490,9 +490,9 @@ asynchronous_metric_log
- 300000 + 60000 @@ -1813,7 +1813,7 @@ stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') Predicted labels have to be in \[-1, 1\]. -1. Predicting +**2.** Predicting diff --git a/docs/en/sql-reference/data-types/aggregatefunction.md b/docs/en/sql-reference/data-types/aggregatefunction.md index f214db20ea7..ce810f9b2a9 100644 --- a/docs/en/sql-reference/data-types/aggregatefunction.md +++ b/docs/en/sql-reference/data-types/aggregatefunction.md @@ -28,7 +28,7 @@ CREATE TABLE t ) ENGINE = ... ``` -[uniq](../../sql-reference/aggregate-functions/reference.md#agg_function-uniq), anyIf ([any](../../sql-reference/aggregate-functions/reference.md#agg_function-any)+[If](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-if)) and [quantiles](../../sql-reference/aggregate-functions/reference.md) are the aggregate functions supported in ClickHouse. +[uniq](../../sql-reference/aggregate-functions/reference.md#agg_function-uniq), anyIf ([any](../../sql-reference/aggregate-functions/reference.md#agg_function-any)+[If](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-if)) and [quantiles](../../sql-reference/aggregate-functions/reference.md#quantiles) are the aggregate functions supported in ClickHouse. ## Usage {#usage} diff --git a/docs/zh/sql-reference/aggregate-functions/combinators.md b/docs/zh/sql-reference/aggregate-functions/combinators.md index c5c3e8a9577..c458097a5fb 100644 --- a/docs/zh/sql-reference/aggregate-functions/combinators.md +++ b/docs/zh/sql-reference/aggregate-functions/combinators.md @@ -1,51 +1,49 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 -toc_title: "\u7EC4\u5408\u5668" +toc_title: 聚合函数组合器 --- # 聚合函数组合器 {#aggregate_functions_combinators} 聚合函数的名称可以附加一个后缀。 这改变了聚合函数的工作方式。 -## -如果 {#agg-functions-combinator-if} +## -If {#agg-functions-combinator-if} -The suffix -If can be appended to the name of any aggregate function. In this case, the aggregate function accepts an extra argument – a condition (Uint8 type). The aggregate function processes only the rows that trigger the condition. If the condition was not triggered even once, it returns a default value (usually zeros or empty strings). +-If可以加到任何聚合函数之后。加了-If之后聚合函数需要接受一个额外的参数，一个条件（Uint8类型），如果条件满足，那聚合函数处理当前的行数据，如果不满足，那返回默认值（通常是0或者空字符串）。 -例: `sumIf(column, cond)`, `countIf(cond)`, `avgIf(x, cond)`, `quantilesTimingIf(level1, level2)(x, cond)`, `argMinIf(arg, val, cond)` 等等。 +例： `sumIf(column, cond)`, `countIf(cond)`, `avgIf(x, cond)`, `quantilesTimingIf(level1, level2)(x, cond)`, `argMinIf(arg, val, cond)` 等等。 -使用条件聚合函数，您可以一次计算多个条件的聚合，而无需使用子查询和 `JOIN`例如，在Yandex的。Metrica，条件聚合函数用于实现段比较功能。 +使用条件聚合函数，您可以一次计算多个条件的聚合，而无需使用子查询和 `JOIN`例如，在Yandex.Metrica，条件聚合函数用于实现段比较功能。 -## -阵列 {#agg-functions-combinator-array} +## -Array {#agg-functions-combinator-array} -Array后缀可以附加到任何聚合函数。 在这种情况下，聚合函数采用的参数 ‘Array(T)’ 类型（数组）而不是 ‘T’ 类型参数。 如果聚合函数接受多个参数，则它必须是长度相等的数组。 在处理数组时，聚合函数的工作方式与所有数组元素的原始聚合函数类似。 -示例1: `sumArray(arr)` -总计所有的所有元素 ‘arr’ 阵列。 在这个例子中，它可以更简单地编写: `sum(arraySum(arr))`. +示例1： `sumArray(arr)` -总计所有的所有元素 ‘arr’ 阵列。在这个例子中，它可以更简单地编写: `sum(arraySum(arr))`. -示例2: `uniqArray(arr)` – Counts the number of unique elements in all ‘arr’ 阵列。 这可以做一个更简单的方法: `uniq(arrayJoin(arr))`，但它并不总是可以添加 ‘arrayJoin’ 到查询。 +示例2： `uniqArray(arr)` – 计算‘arr’中唯一元素的个数。这可以是一个更简单的方法： `uniq(arrayJoin(arr))`，但它并不总是可以添加 ‘arrayJoin’ 到查询。 --如果和-阵列可以组合。 然而, ‘Array’ 必须先来，然后 ‘If’. 例: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. 由于这个顺序，该 ‘cond’ 参数不会是数组。 +如果和-If组合，‘Array’ 必须先来，然后 ‘If’. 例： `uniqArrayIf(arr, cond)`， `quantilesTimingArrayIf(level1, level2)(arr, cond)`。由于这个顺序，该 ‘cond’ 参数不会是数组。 -## -州 {#agg-functions-combinator-state} +## -State {#agg-functions-combinator-state} -如果应用此combinator，则聚合函数不会返回结果值（例如唯一值的数量 [uniq](reference.md#agg_function-uniq) 函数），但聚合的中间状态（用于 `uniq`，这是用于计算唯一值的数量的散列表）。 这是一个 `AggregateFunction(...)` 可用于进一步处理或存储在表中以完成聚合。 +如果应用此combinator，则聚合函数不会返回结果值（例如唯一值的数量 [uniq](reference.md#agg_function-uniq) 函数），但是返回聚合的中间状态（对于 `uniq`，返回的是计算唯一值的数量的哈希表）。 这是一个 `AggregateFunction(...)` 可用于进一步处理或存储在表中以完成稍后的聚合。 要使用这些状态，请使用: - [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) 表引擎。 -- [最后聚会](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) 功能。 -- [跑累积](../../sql-reference/functions/other-functions.md#function-runningaccumulate) 功能。 -- [-合并](#aggregate_functions_combinators-merge) combinator +- [finalizeAggregation](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) 功能。 +- [runningAccumulate](../../sql-reference/functions/other-functions.md#function-runningaccumulate) 功能。 +- [-Merge](#aggregate_functions_combinators-merge) combinator - [-MergeState](#aggregate_functions_combinators-mergestate) combinator -## -合并 {#aggregate_functions_combinators-merge} +## -Merge {#aggregate_functions_combinators-merge} 如果应用此组合器，则聚合函数将中间聚合状态作为参数，组合状态以完成聚合，并返回结果值。 ## -MergeState {#aggregate_functions_combinators-mergestate} -以与-Merge combinator相同的方式合并中间聚合状态。 但是，它不会返回结果值，而是返回中间聚合状态，类似于-State combinator。 +以与-Merge 相同的方式合并中间聚合状态。 但是，它不会返回结果值，而是返回中间聚合状态，类似于-State。 ## -ForEach {#agg-functions-combinator-foreach} @@ -55,7 +53,7 @@ The suffix -If can be appended to the name of any aggregate function. In this ca 更改聚合函数的行为。 -如果聚合函数没有输入值，则使用此combinator，它返回其返回数据类型的默认值。 适用于可以采用空输入数据的聚合函数。 +如果聚合函数没有输入值，则使用此组合器它返回其返回数据类型的默认值。 适用于可以采用空输入数据的聚合函数。 `-OrDefault` 可与其他组合器一起使用。 @@ -67,7 +65,7 @@ The suffix -If can be appended to the name of any aggregate function. In this ca **参数** -- `x` — Aggregate function parameters. +- `x` — 聚合函数参数。 **返回值** @@ -174,7 +172,7 @@ FROM └────────────────────────────────┘ ``` -## -重新采样 {#agg-functions-combinator-resample} +## -Resample {#agg-functions-combinator-resample} 允许您将数据划分为组，然后单独聚合这些组中的数据。 通过将一列中的值拆分为间隔来创建组。 @@ -184,19 +182,19 @@ FROM **参数** -- `start` — Starting value of the whole required interval for `resampling_key` 值。 -- `stop` — Ending value of the whole required interval for `resampling_key` 值。 整个时间间隔不包括 `stop` 价值 `[start, stop)`. -- `step` — Step for separating the whole interval into subintervals. The `aggFunction` 在每个子区间上独立执行。 -- `resampling_key` — Column whose values are used for separating data into intervals. +- `start` — `resampling_key` 开始值。 +- `stop` — `resampling_key` 结束边界。 区间内部不包含 `stop` 值，即 `[start, stop)`. +- `step` — 分组的步长。 The `aggFunction` 在每个子区间上独立执行。 +- `resampling_key` — 取样列，被用来分组. - `aggFunction_params` — `aggFunction` 参数。 **返回值** -- 阵列 `aggFunction` 每个子区间的结果。 +- `aggFunction` 每个子区间的结果，结果为数组。 **示例** -考虑一下 `people` 具有以下数据的表: +考虑一下 `people` 表具有以下数据的表结构： ``` text ┌─name───┬─age─┬─wage─┐ @@ -209,9 +207,9 @@ FROM └────────┴─────┴──────┘ ``` -让我们得到的人的名字，他们的年龄在于的时间间隔 `[30,60)` 和 `[60,75)`. 由于我们使用整数表示的年龄，我们得到的年龄 `[30, 59]` 和 `[60,74]` 间隔。 +让我们得到的人的名字，他们的年龄在于的时间间隔 `[30,60)` 和 `[60,75)`。 由于我们使用整数表示的年龄，我们得到的年龄 `[30, 59]` 和 `[60,74]` 间隔。 -要在数组中聚合名称，我们使用 [groupArray](reference.md#agg_function-grouparray) 聚合函数。 这需要一个参数。 在我们的例子中，它是 `name` 列。 该 `groupArrayResample` 函数应该使用 `age` 按年龄聚合名称的列。 要定义所需的时间间隔，我们通过 `30, 75, 30` 参数到 `groupArrayResample` 功能。 +要在数组中聚合名称，我们使用 [groupArray](reference.md#agg_function-grouparray) 聚合函数。 这需要一个参数。 在我们的例子中，它是 `name` 列。 `groupArrayResample` 函数应该使用 `age` 按年龄聚合名称， 要定义所需的时间间隔，我们传入 `30, 75, 30` 参数给 `groupArrayResample` 函数。 ``` sql SELECT groupArrayResample(30, 75, 30)(name, age) FROM people @@ -225,7 +223,7 @@ SELECT groupArrayResample(30, 75, 30)(name, age) FROM people 考虑结果。 -`Jonh` 是因为他太年轻了 其他人按照指定的年龄间隔进行分配。 +`Jonh` 没有被选中，因为他太年轻了。 其他人按照指定的年龄间隔进行分配。 现在让我们计算指定年龄间隔内的总人数和平均工资。 diff --git a/docs/zh/sql-reference/aggregate-functions/index.md b/docs/zh/sql-reference/aggregate-functions/index.md index 06666c49d03..57d8e362d99 100644 --- a/docs/zh/sql-reference/aggregate-functions/index.md +++ b/docs/zh/sql-reference/aggregate-functions/index.md @@ -1,9 +1,6 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u805A\u5408\u51FD\u6570" toc_priority: 33 -toc_title: "\u5BFC\u8A00" +toc_title: 简介 --- # 聚合函数 {#aggregate-functions} diff --git a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md index 830581beba7..69572086549 100644 --- a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md @@ -1,15 +1,13 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 38 -toc_title: "\u53C2\u6570" +toc_title: 参数聚合函数 --- # 参数聚合函数 {#aggregate_functions_parametric} -Some aggregate functions can accept not only argument columns (used for compression), but a set of parameters – constants for initialization. The syntax is two pairs of brackets instead of one. The first is for parameters, and the second is for arguments. +一些聚合函数不仅可以接受参数列（用于压缩），也可以接收常量的初始化参数。这种语法是接受两个括号的参数，第一个数初始化参数，第二个是入参。 -## 直方图 {#histogram} +## histogram {#histogram} 计算自适应直方图。 它不能保证精确的结果。 @@ -21,20 +19,21 @@ histogram(number_of_bins)(values) **参数** -`number_of_bins` — Upper limit for the number of bins in the histogram. The function automatically calculates the number of bins. It tries to reach the specified number of bins, but if it fails, it uses fewer bins. -`values` — [表达式](../syntax.md#syntax-expressions) 导致输入值。 +`number_of_bins` — 直方图bin个数，这个函数会自动计算bin的数量，而且会尽量使用指定值，如果无法做到，那就使用更小的bin个数。 + +`values` — [表达式](../syntax.md#syntax-expressions) 输入值。 **返回值** -- [阵列](../../sql-reference/data-types/array.md) 的 [元组](../../sql-reference/data-types/tuple.md) 下面的格式: +- [Array](../../sql-reference/data-types/array.md) 的 [Tuples](../../sql-reference/data-types/tuple.md) 如下： ``` [(lower_1, upper_1, height_1), ... (lower_N, upper_N, height_N)] ``` - - `lower` — Lower bound of the bin. - - `upper` — Upper bound of the bin. - - `height` — Calculated height of the bin. + - `lower` — bin的下边界。 + - `upper` — bin的上边界。 + - `height` — bin的计算权重。 **示例** @@ -53,7 +52,7 @@ FROM ( └─────────────────────────────────────────────────────────────────────────┘ ``` -您可以使用 [酒吧](../../sql-reference/functions/other-functions.md#function-bar) 功能，例如: +您可以使用 [bar](../../sql-reference/functions/other-functions.md#function-bar) 功能，例如: ``` sql WITH histogram(5)(rand() % 100) AS hist @@ -93,11 +92,11 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) **参数** -- `pattern` — Pattern string. See [模式语法](#sequence-function-pattern-syntax). +- `pattern` — 模式字符串。 参考 [模式语法](#sequence-function-pattern-syntax). -- `timestamp` — Column considered to contain time data. Typical data types are `Date` 和 `DateTime`. 您还可以使用任何支持的 [UInt](../../sql-reference/data-types/int-uint.md) 数据类型。 +- `timestamp` — 包含时间的列。典型的时间类型是： `Date` 和 `DateTime`。您还可以使用任何支持的 [UInt](../../sql-reference/data-types/int-uint.md) 数据类型。 -- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. 最多可以传递32个条件参数。 该函数只考虑这些条件中描述的事件。 如果序列包含未在条件中描述的数据，则函数将跳过这些数据。 +- `cond1`, `cond2` — 事件链的约束条件。 数据类型是： `UInt8`。 最多可以传递32个条件参数。 该函数只考虑这些条件中描述的事件。 如果序列包含未在条件中描述的数据，则函数将跳过这些数据。 **返回值** @@ -109,11 +108,11 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) **模式语法** -- `(?N)` — Matches the condition argument at position `N`. 条件在编号 `[1, 32]` 范围。 例如, `(?1)` 匹配传递给 `cond1` 参数。 +- `(?N)` — 在位置`N`匹配条件参数。 条件在编号 `[1, 32]` 范围。 例如, `(?1)` 匹配传递给 `cond1` 参数。 -- `.*` — Matches any number of events. You don't need conditional arguments to match this element of the pattern. +- `.*` — 匹配任何事件的数字。 不需要条件参数来匹配这个模式。 -- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` 匹配彼此发生超过1800秒的事件。 这些事件之间可以存在任意数量的任何事件。 您可以使用 `>=`, `>`, `<`, `<=` 运营商。 +- `(?t operator value)` — 分开两个事件的时间。 例如： `(?1)(?t>1800)(?2)` 匹配彼此发生超过1800秒的事件。 这些事件之间可以存在任意数量的任何事件。 您可以使用 `>=`, `>`, `<`, `<=` 运算符。 **例** @@ -169,7 +168,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM ## sequenceCount(pattern)(time, cond1, cond2, …) {#function-sequencecount} -计数与模式匹配的事件链的数量。 该函数搜索不重叠的事件链。 当前链匹配后，它开始搜索下一个链。 +计算与模式匹配的事件链的数量。该函数搜索不重叠的事件链。当前链匹配后，它开始搜索下一个链。 !!! warning "警告" 在同一秒钟发生的事件可能以未定义的顺序排列在序列中，影响结果。 @@ -180,11 +179,11 @@ sequenceCount(pattern)(timestamp, cond1, cond2, ...) **参数** -- `pattern` — Pattern string. See [模式语法](#sequence-function-pattern-syntax). +- `pattern` — 模式字符串。 参考：[模式语法](#sequence-function-pattern-syntax). -- `timestamp` — Column considered to contain time data. Typical data types are `Date` 和 `DateTime`. 您还可以使用任何支持的 [UInt](../../sql-reference/data-types/int-uint.md) 数据类型。 +- `timestamp` — 包含时间的列。典型的时间类型是： `Date` 和 `DateTime`。您还可以使用任何支持的 [UInt](../../sql-reference/data-types/int-uint.md) 数据类型。 -- `cond1`, `cond2` — Conditions that describe the chain of events. Data type: `UInt8`. 最多可以传递32个条件参数。 该函数只考虑这些条件中描述的事件。 如果序列包含未在条件中描述的数据，则函数将跳过这些数据。 +- `cond1`, `cond2` — 事件链的约束条件。 数据类型是： `UInt8`。 最多可以传递32个条件参数。该函数只考虑这些条件中描述的事件。 如果序列包含未在条件中描述的数据，则函数将跳过这些数据。 **返回值** @@ -227,9 +226,9 @@ SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t 搜索滑动时间窗中的事件链，并计算从链中发生的最大事件数。 -该函数根据算法工作: +该函数采用如下算法： -- 该函数搜索触发链中的第一个条件并将事件计数器设置为1的数据。 这是滑动窗口启动的时刻。 +- 该函数搜索触发链中的第一个条件并将事件计数器设置为1。 这是滑动窗口启动的时刻。 - 如果来自链的事件在窗口内顺序发生，则计数器将递增。 如果事件序列中断，则计数器不会增加。 @@ -243,11 +242,11 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) **参数** -- `window` — Length of the sliding window in seconds. -- `mode` -这是一个可选的参数。 - - `'strict'` -当 `'strict'` 设置时，windowFunnel()仅对唯一值应用条件。 -- `timestamp` — Name of the column containing the timestamp. Data types supported: [日期](../../sql-reference/data-types/date.md), [日期时间](../../sql-reference/data-types/datetime.md#data_type-datetime) 和其他无符号整数类型（请注意，即使时间戳支持 `UInt64` 类型，它的值不能超过Int64最大值，即2^63-1）。 -- `cond` — Conditions or data describing the chain of events. [UInt8](../../sql-reference/data-types/int-uint.md). +- `window` — 滑动窗户的大小，单位是秒。 +- `mode` - 这是一个可选的参数。 + - `'strict'` - 当 `'strict'` 设置时，windowFunnel()仅对唯一值应用匹配条件。 +- `timestamp` — 包含时间的列。 数据类型支持： [日期](../../sql-reference/data-types/date.md), [日期时间](../../sql-reference/data-types/datetime.md#data_type-datetime) 和其他无符号整数类型（请注意，即使时间戳支持 `UInt64` 类型，它的值不能超过Int64最大值，即2^63-1）。 +- `cond` — 事件链的约束条件。 [UInt8](../../sql-reference/data-types/int-uint.md) 类型。 **返回值** @@ -284,7 +283,7 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) └────────────┴─────────┴─────────────────────┴─────────┴─────────┘ ``` -了解用户有多远 `user_id` 可以在2019的1-2月期间通过链条。 +了解用户`user_id` 可以在2019的1-2月期间通过链条多远。 查询: @@ -315,10 +314,10 @@ ORDER BY level ASC ## Retention {#retention} -该函数将一组条件作为参数，类型为1到32个参数 `UInt8` 表示事件是否满足特定条件。 +该函数将一组条件作为参数，类型为1到32个 `UInt8` 类型的参数，用来表示事件是否满足特定条件。 任何条件都可以指定为参数（如 [WHERE](../../sql-reference/statements/select/where.md#select-where)). -除了第一个以外，条件成对适用：如果第一个和第二个是真的，第二个结果将是真的，如果第一个和fird是真的，第三个结果将是真的，等等。 +除了第一个以外，条件成对适用：如果第一个和第二个是真的，第二个结果将是真的，如果第一个和第三个是真的，第三个结果将是真的，等等。 **语法** @@ -328,22 +327,22 @@ retention(cond1, cond2, ..., cond32); **参数** -- `cond` — an expression that returns a `UInt8` 结果（1或0）。 +- `cond` — 返回 `UInt8` 结果（1或0）的表达式。 **返回值** 数组为1或0。 -- 1 — condition was met for the event. -- 0 — condition wasn't met for the event. +- 1 — 条件满足。 +- 0 — 条件不满足。 类型: `UInt8`. **示例** -让我们考虑计算的一个例子 `retention` 功能，以确定网站流量。 +让我们考虑使用 `retention` 功能的一个例子 ，以确定网站流量。 -**1.** Сreate a table to illustrate an example. +**1.** 举例说明，先创建一张表。 ``` sql CREATE TABLE retention_test(date Date, uid Int32) ENGINE = Memory; @@ -402,7 +401,7 @@ SELECT * FROM retention_test └────────────┴─────┘ ``` -**2.** 按唯一ID对用户进行分组 `uid` 使用 `retention` 功能。 +**2.** 按唯一ID `uid` 对用户进行分组，使用 `retention` 功能。 查询: @@ -466,7 +465,7 @@ FROM └────┴────┴────┘ ``` -哪里: +条件: - `r1`-2020-01-01期间访问该网站的独立访问者数量（ `cond1` 条件）。 - `r2`-在2020-01-01和2020-01-02之间的特定时间段内访问该网站的唯一访问者的数量 (`cond1` 和 `cond2` 条件）。 @@ -474,9 +473,9 @@ FROM ## uniqUpTo(N)(x) {#uniquptonx} -Calculates the number of different argument values ​​if it is less than or equal to N. If the number of different argument values is greater than N, it returns N + 1. +计算小于或者等于N的不同参数的个数。如果结果大于N，那返回N+1。 -建议使用小Ns，高达10。 N的最大值为100。 +建议使用较小的Ns，比如：10。N的最大值为100。 对于聚合函数的状态，它使用的内存量等于1+N\*一个字节值的大小。 对于字符串，它存储8个字节的非加密哈希。 也就是说，计算是近似的字符串。 @@ -488,12 +487,12 @@ Calculates the number of different argument values ​​if it is less than or e 用法示例: ``` text -Problem: Generate a report that shows only keywords that produced at least 5 unique users. -Solution: Write in the GROUP BY query SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5 +问题：产出一个不少于五个唯一用户的关键字报告 +解决方案： 写group by查询语句 HAVING uniqUpTo(4)(UserID) >= 5 ``` +## sumMapFiltered(keys\_to\_keep)(keys, values) {#summapfilteredkeys-to-keepkeys-values} + +和 [sumMap](reference.md#agg_functions-summap) 基本一致， 除了一个键数组作为参数传递。这在使用高基数key时尤其有用。 + [原始文章](https://clickhouse.tech/docs/en/query_language/agg_functions/parametric_functions/) - -## sumMapFiltered(keys\_to\_keep)(键值) {#summapfilteredkeys-to-keepkeys-values} - -同样的行为 [sumMap](reference.md#agg_functions-summap) 除了一个键数组作为参数传递。 这在使用高基数密钥时尤其有用。 diff --git a/docs/zh/sql-reference/aggregate-functions/reference.md b/docs/zh/sql-reference/aggregate-functions/reference.md index 53510900536..7d5ecda7bb4 100644 --- a/docs/zh/sql-reference/aggregate-functions/reference.md +++ b/docs/zh/sql-reference/aggregate-functions/reference.md @@ -1,13 +1,11 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 36 -toc_title: "\u53C2\u8003\u8D44\u6599" +toc_title: 聚合函数 --- # 聚合函数引用 {#aggregate-functions-reference} -## 计数 {#agg_function-count} +## count {#agg_function-count} 计数行数或非空值。 @@ -73,7 +71,7 @@ SELECT count(DISTINCT num) FROM t 这个例子表明 `count(DISTINCT num)` 由执行 `uniqExact` 根据功能 `count_distinct_implementation` 设定值。 -## 任何(x) {#agg_function-any} +## any(x) {#agg_function-any} 选择第一个遇到的值。 查询可以以任何顺序执行，甚至每次都以不同的顺序执行，因此此函数的结果是不确定的。 @@ -115,7 +113,7 @@ FROM ontime 选择遇到的最后一个值。 其结果是一样不确定的 `any` 功能。 -## 集团比特 {#groupbitand} +## groupBitAnd {#groupbitand} 按位应用 `AND` 对于一系列的数字。 @@ -337,7 +335,7 @@ SELECT argMin(user, salary) FROM salary 总计 ‘value’ 数组根据在指定的键 ‘key’ 阵列。 传递键和值数组的元组与传递两个键和值数组是同义的。 元素的数量 ‘key’ 和 ‘value’ 总计的每一行必须相同。 -Returns a tuple of two arrays: keys in sorted order, and values ​​summed for the corresponding keys. +返回两个数组的一个二元组： key是排好序的，value是对应key的求和。 示例: @@ -374,7 +372,7 @@ GROUP BY timeslot ## skewPop {#skewpop} -计算 [歪斜](https://en.wikipedia.org/wiki/Skewness) 的序列。 +计算的序列[偏度](https://en.wikipedia.org/wiki/Skewness)。 ``` sql skewPop(expr) @@ -386,7 +384,7 @@ skewPop(expr) **返回值** -The skewness of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md) +给定序列的偏度。类型 — [Float64](../../sql-reference/data-types/float.md) **示例** @@ -410,7 +408,7 @@ skewSamp(expr) **返回值** -The skewness of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). 如果 `n <= 1` (`n` 是样本的大小），则该函数返回 `nan`. +给定序列的偏度。 类型 — [Float64](../../sql-reference/data-types/float.md). 如果 `n <= 1` (`n` 是样本的大小），则该函数返回 `nan`. **示例** @@ -432,7 +430,7 @@ kurtPop(expr) **返回值** -The kurtosis of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md) +给定序列的峰度。 类型 — [Float64](../../sql-reference/data-types/float.md) **示例** @@ -456,7 +454,7 @@ kurtSamp(expr) **返回值** -The kurtosis of the given distribution. Type — [Float64](../../sql-reference/data-types/float.md). 如果 `n <= 1` (`n` 是样本的大小），则该函数返回 `nan`. +给定序列的峰度。类型 — [Float64](../../sql-reference/data-types/float.md). 如果 `n <= 1` (`n` 是样本的大小），则该函数返回 `nan`. **示例** @@ -533,7 +531,7 @@ FROM ( 只适用于数字。 结果总是Float64。 -## 平均加权 {#avgweighted} +## avgWeighted {#avgweighted} 计算 [加权算术平均值](https://en.wikipedia.org/wiki/Weighted_arithmetic_mean). @@ -545,10 +543,10 @@ avgWeighted(x, weight) **参数** -- `x` — Values. [整数](../data-types/int-uint.md) 或 [浮点](../data-types/float.md). -- `weight` — Weights of the values. [整数](../data-types/int-uint.md) 或 [浮点](../data-types/float.md). +- `x` — 值。 [整数](../data-types/int-uint.md) 或 [浮点](../data-types/float.md). +- `weight` — 值的加权。 [整数](../data-types/int-uint.md) 或 [浮点](../data-types/float.md). -类型 `x` 和 `weight` 一定是一样的 +`x` 和 `weight` 的类型一定是一样的 **返回值** @@ -590,7 +588,7 @@ uniq(x[, ...]) - A [UInt64](../../sql-reference/data-types/int-uint.md)-键入号码。 -**实施细节** +**实现细节** 功能: @@ -598,7 +596,7 @@ uniq(x[, ...]) - 使用自适应采样算法。 对于计算状态，该函数使用最多65536个元素哈希值的样本。 - This algorithm is very accurate and very efficient on the CPU. When the query contains several of these functions, using `uniq` is almost as fast as using other aggregate functions. + 这个算法是非常精确的，并且对于CPU来说非常高效。如果查询包含一些这样的函数，那和其他聚合函数相比 `uniq` 将是几乎一样快。 - 确定性地提供结果（它不依赖于查询处理顺序）。 @@ -629,17 +627,17 @@ uniqCombined(HLL_precision)(x[, ...]) **返回值** -- 一个数字 [UInt64](../../sql-reference/data-types/int-uint.md)-键入号码。 +- 一个[UInt64](../../sql-reference/data-types/int-uint.md)类型的数字。 -**实施细节** +**实现细节** 功能: - 计算散列（64位散列 `String` 否则32位）对于聚合中的所有参数，然后在计算中使用它。 -- 使用三种算法的组合：数组、哈希表和HyperLogLog与error错表。 +- 使用三种算法的组合：数组、哈希表和包含错误修正表的HyperLogLog。 - For a small number of distinct elements, an array is used. When the set size is larger, a hash table is used. For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory. + 少量的不同的值，使用数组。 值再多一些，使用哈希表。对于大量的数据来说，使用HyperLogLog，HyperLogLog占用一个固定的内存空间。 - 确定性地提供结果（它不依赖于查询处理顺序）。 @@ -650,7 +648,7 @@ uniqCombined(HLL_precision)(x[, ...]) - 消耗少几倍的内存。 - 计算精度高出几倍。 -- 通常具有略低的性能。 在某些情况下, `uniqCombined` 可以表现得比 `uniq`，例如，使用通过网络传输大量聚合状态的分布式查询。 +- 通常具有略低的性能。 在某些情况下, `uniqCombined` 可以表现得比 `uniq` 好，例如，使用通过网络传输大量聚合状态的分布式查询。 **另请参阅** @@ -679,7 +677,7 @@ uniqHLL12(x[, ...]) - A [UInt64](../../sql-reference/data-types/int-uint.md)-键入号码。 -**实施细节** +**实现细节** 功能: @@ -707,9 +705,9 @@ uniqHLL12(x[, ...]) uniqExact(x[, ...]) ``` -使用 `uniqExact` 功能，如果你绝对需要一个确切的结果。 否则使用 [uniq](#agg_function-uniq) 功能。 +如果你绝对需要一个确切的结果，使用 `uniqExact` 功能。 否则使用 [uniq](#agg_function-uniq) 功能。 -该 `uniqExact` 功能使用更多的内存比 `uniq`，因为状态的大小随着不同值的数量的增加而无界增长。 +`uniqExact` 比 `uniq` 使用更多的内存，因为状态的大小随着不同值的数量的增加而无界增长。 **参数** @@ -721,7 +719,7 @@ uniqExact(x[, ...]) - [uniqCombined](#agg_function-uniqcombined) - [uniqHLL12](#agg_function-uniqhll12) -## 群交(x),群交(max\_size)(x) {#agg_function-grouparray} +## groupArray(x), groupArray(max\_size)(x) {#agg_function-grouparray} 创建参数值的数组。 值可以按任何（不确定）顺序添加到数组中。 @@ -748,10 +746,10 @@ groupArrayInsertAt(default_x, size)(x, pos); **参数** -- `x` — Value to be inserted. [表达式](../syntax.md#syntax-expressions) 导致的一个 [支持的数据类型](../../sql-reference/data-types/index.md). -- `pos` — Position at which the specified element `x` 将被插入。 数组中的索引编号从零开始。 [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). -- `default_x`— Default value for substituting in empty positions. Optional parameter. [表达式](../syntax.md#syntax-expressions) 导致为配置的数据类型 `x` 参数。 如果 `default_x` 未定义，则 [默认值](../../sql-reference/statements/create.md#create-default-values) 被使用。 -- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value `default_x` 必须指定。 [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `x` — 被插入的值。[表达式](../syntax.md#syntax-expressions) 导致的一个 [支持的数据类型](../../sql-reference/data-types/index.md). +- `pos` — `x` 将被插入的位置。 数组中的索引编号从零开始。 [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `default_x`— 如果代入值为空，则使用默认值。可选参数。[表达式](../syntax.md#syntax-expressions) 为 `x` 数据类型的数据。 如果 `default_x` 未定义，则 [默认值](../../sql-reference/statements/create.md#create-default-values) 被使用。 +- `size`— 结果数组的长度。可选参数。如果使用该参数，`default_x` 必须指定。 [UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges). **返回值** @@ -803,7 +801,7 @@ SELECT groupArrayInsertAt('-', 5)(toString(number), number * 2) FROM numbers(5); └───────────────────────────────────────────────────────────────────┘ ``` -元件的多线程插入到一个位置。 +在一个位置多线程插入数据。 查询: @@ -832,8 +830,8 @@ groupArrayMovingSum(window_size)(numbers_for_summing) **参数** -- `numbers_for_summing` — [表达式](../syntax.md#syntax-expressions) 生成数值数据类型值。 -- `window_size` — Size of the calculation window. +- `numbers_for_summing` — [表达式](../syntax.md#syntax-expressions) 为数值数据类型值。 +- `window_size` — 窗口大小。 **返回值** @@ -906,13 +904,13 @@ groupArrayMovingAvg(window_size)(numbers_for_summing) **参数** - `numbers_for_summing` — [表达式](../syntax.md#syntax-expressions) 生成数值数据类型值。 -- `window_size` — Size of the calculation window. +- `window_size` — 窗口大小。 **返回值** - 与输入数据大小和类型相同的数组。 -该函数使用 [四舍五入到零](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero). 它截断结果数据类型的小数位数。 +该函数使用 [四舍五入到零](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero). 它截断无意义的小数位来保证结果的数据类型。 **示例** @@ -967,20 +965,20 @@ FROM t └───────────┴──────────────────────────────────┴───────────────────────┘ ``` -## 禄,赂麓ta脌麓,):脡,,拢脢,group媒group)galaxy s8碌胫脢)禄煤)酶脱脩) {#groupuniqarrayx-groupuniqarraymax-sizex} +## groupUniqArray(x), groupUniqArray(max\_size)(x) {#groupuniqarrayx-groupuniqarraymax-sizex} 从不同的参数值创建一个数组。 内存消耗是一样的 `uniqExact` 功能。 -第二个版本（与 `max_size` 参数）将结果数组的大小限制为 `max_size` 元素。 +第二个版本（`max_size` 参数）将结果数组的大小限制为 `max_size` 元素。 例如, `groupUniqArray(1)(x)` 相当于 `[any(x)]`. -## 分位数 {#quantile} +## quantile {#quantile} -计算近似值 [分位数](https://en.wikipedia.org/wiki/Quantile) 的数字数据序列。 +计算数字序列的近似[分位数](https://en.wikipedia.org/wiki/Quantile)。 -此功能适用 [油藏采样](https://en.wikipedia.org/wiki/Reservoir_sampling) 随着储存器大小高达8192和随机数发生器进行采样。 结果是非确定性的。 要获得精确的分位数，请使用 [quantileExact](#quantileexact) 功能。 +此功能适用 [水塘抽样(](https://en.wikipedia.org/wiki/Reservoir_sampling)，使用储存器最大到8192和随机数发生器进行采样。 结果是非确定性的。 要获得精确的分位数，请使用 [quantileExact](#quantileexact) 功能。 -当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。 +当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。 **语法** @@ -992,12 +990,12 @@ quantile(level)(expr) **参数** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md). +- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). +- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。 **返回值** -- 指定电平的近似分位数。 +- 指定层次的近似分位数。 类型: @@ -1037,13 +1035,13 @@ SELECT quantile(val) FROM t - [中位数](#median) - [分位数](#quantiles) -## 量化确定 {#quantiledeterministic} +## quantileDeterministic {#quantiledeterministic} -计算近似值 [分位数](https://en.wikipedia.org/wiki/Quantile) 的数字数据序列。 +计算数字序列的近似[分位数](https://en.wikipedia.org/wiki/Quantile)。 -此功能适用 [油藏采样](https://en.wikipedia.org/wiki/Reservoir_sampling) 与储层大小高达8192和采样的确定性算法。 结果是确定性的。 要获得精确的分位数，请使用 [quantileExact](#quantileexact) 功能。 +此功能适用 [水塘抽样(](https://en.wikipedia.org/wiki/Reservoir_sampling)，使用储存器最大到8192和随机数发生器进行采样。 结果是非确定性的。 要获得精确的分位数，请使用 [quantileExact](#quantileexact) 功能。 -当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。 +当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。 **语法** @@ -1055,13 +1053,13 @@ quantileDeterministic(level)(expr, determinator) **参数** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md). -- `determinator` — Number whose hash is used instead of a random number generator in the reservoir sampling algorithm to make the result of sampling deterministic. As a determinator you can use any deterministic positive number, for example, a user id or an event id. If the same determinator value occures too often, the function works incorrectly. +- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). +- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。 +- `determinator` — 一个数字，其hash被用来代替在水塘抽样中随机生成的数字，这样可以保证取样的确定性。你可以使用用户ID或者事件ID等任何正数，但是如果相同的 `determinator` 出现多次，那结果很可能不正确。 **返回值** -- 指定电平的近似分位数。 +- 指定层次的近似分位数。 类型: @@ -1103,11 +1101,11 @@ SELECT quantileDeterministic(val, 1) FROM t ## quantileExact {#quantileexact} -正是计算 [分位数](https://en.wikipedia.org/wiki/Quantile) 的数字数据序列。 +准确计算数字序列的[分位数](https://en.wikipedia.org/wiki/Quantile)。 -To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` 内存，其中 `n` 是传递的多个值。 然而，对于少量的值，该函数是非常有效的。 +为了准确计算，所有输入的数据被合并为一个数组，并且部分的排序。因此该函数需要 `O(n)` 的内存，n为输入数据的个数。但是对于少量数据来说，该函数还是非常有效的。 -当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。 +当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。 **语法** @@ -1119,12 +1117,12 @@ quantileExact(level)(expr) **参数** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md). +- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). +- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。 **返回值** -- 指定电平的分位数。 +- 指定层次的分位数。 类型: @@ -1153,13 +1151,13 @@ SELECT quantileExact(number) FROM numbers(10) - [中位数](#median) - [分位数](#quantiles) -## 分位数加权 {#quantileexactweighted} +## quantileExactWeighted {#quantileexactweighted} -正是计算 [分位数](https://en.wikipedia.org/wiki/Quantile) 数值数据序列，考虑到每个元素的权重。 +考虑到每个元素的权重，然后准确计算数值序列的[分位数](https://en.wikipedia.org/wiki/Quantile)。 -To get exact value, all the passed values ​​are combined into an array, which is then partially sorted. Each value is counted with its weight, as if it is present `weight` times. A hash table is used in the algorithm. Because of this, if the passed values ​​are frequently repeated, the function consumes less RAM than [quantileExact](#quantileexact). 您可以使用此功能，而不是 `quantileExact` 并指定重量1。 +为了准确计算，所有输入的数据被合并为一个数组，并且部分的排序。每个输入值需要根据 `weight` 计算求和。该算法使用哈希表。正因为如此，在数据重复较多的时候使用的内存是少于[quantileExact](#quantileexact)的。 您可以使用此函数代替 `quantileExact` 并指定重量1。 -当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。 +当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。 **语法** @@ -1171,13 +1169,13 @@ quantileExactWeighted(level)(expr, weight) **参数** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md). -- `weight` — Column with weights of sequence members. Weight is a number of value occurrences. +- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). +- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。 +- `weight` — 权重序列。 权重是一个数据出现的数值。 **返回值** -- 指定电平的分位数。 +- 指定层次的分位数。 类型: @@ -1217,13 +1215,13 @@ SELECT quantileExactWeighted(n, val) FROM t - [中位数](#median) - [分位数](#quantiles) -## 分位定时 {#quantiletiming} +## quantileTiming {#quantiletiming} -随着确定的精度计算 [分位数](https://en.wikipedia.org/wiki/Quantile) 的数字数据序列。 +使用确定的精度计算数字数据序列的[分位数](https://en.wikipedia.org/wiki/Quantile)。 结果是确定性的（它不依赖于查询处理顺序）。 该函数针对描述加载网页时间或后端响应时间等分布的序列进行了优化。 -当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。 +当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。 **语法** @@ -1235,12 +1233,12 @@ quantileTiming(level)(expr) **参数** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). +- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). -- `expr` — [表达式](../syntax.md#syntax-expressions) 在一个列值返回 [浮动\*](../../sql-reference/data-types/float.md)-键入号码。 +- `expr` — [表达式](../syntax.md#syntax-expressions)，返回 [浮动\*](../../sql-reference/data-types/float.md)类型数据。 - - If negative values are passed to the function, the behavior is undefined. - - If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000. + - 如果输入负值，那结果是不可预期的。 + - 如果输入值大于30000（页面加载时间大于30s），那我们假设为30000。 **精度** @@ -1252,16 +1250,16 @@ quantileTiming(level)(expr) 否则，计算结果将四舍五入到16毫秒的最接近倍数。 !!! note "注" - 对于计算页面加载时间分位数，此函数比 [分位数](#quantile). + 对于计算页面加载时间分位数，此函数比 [分位数](#quantile)更有效和准确。 **返回值** -- 指定电平的分位数。 +- 指定层次的分位数。 类型: `Float32`. !!! note "注" - 如果没有值传递给函数（当使用 `quantileTimingIf`), [阿南](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 被返回。 这样做的目的是将这些案例与导致零的案例区分开来。 看 [按条款订购](../statements/select/order-by.md#select-order-by) 对于排序注意事项 `NaN` 值。 + 如果没有值传递给函数（当使用 `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 被返回。 这样做的目的是将这些案例与导致零的案例区分开来。 看 [ORDER BY clause](../statements/select/order-by.md#select-order-by) 对于 `NaN` 值排序注意事项。 **示例** @@ -1300,13 +1298,13 @@ SELECT quantileTiming(response_time) FROM t - [中位数](#median) - [分位数](#quantiles) -## 分位时间加权 {#quantiletimingweighted} +## quantileTimingWeighted {#quantiletimingweighted} -随着确定的精度计算 [分位数](https://en.wikipedia.org/wiki/Quantile) 根据每个序列成员的权重对数字数据序列进行处理。 +根据每个序列成员的权重，使用确定的精度计算数字序列的[分位数](https://en.wikipedia.org/wiki/Quantile)。 结果是确定性的（它不依赖于查询处理顺序）。 该函数针对描述加载网页时间或后端响应时间等分布的序列进行了优化。 -当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。 +当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。 **语法** @@ -1318,14 +1316,14 @@ quantileTimingWeighted(level)(expr, weight) **参数** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). +- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). -- `expr` — [表达式](../syntax.md#syntax-expressions) 在一个列值返回 [浮动\*](../../sql-reference/data-types/float.md)-键入号码。 +- `expr` — [表达式](../syntax.md#syntax-expressions)，返回 [浮动\*](../../sql-reference/data-types/float.md)类型数据。 - - If negative values are passed to the function, the behavior is undefined. - - If the value is greater than 30,000 (a page loading time of more than 30 seconds), it is assumed to be 30,000. + - 如果输入负值，那结果是不可预期的。 + - 如果输入值大于30000（页面加载时间大于30s），那我们假设为30000。 -- `weight` — Column with weights of sequence elements. Weight is a number of value occurrences. +- `weight` — 权重序列。 权重是一个数据出现的数值。 **精度** @@ -1337,16 +1335,16 @@ quantileTimingWeighted(level)(expr, weight) 否则，计算结果将四舍五入到16毫秒的最接近倍数。 !!! note "注" - 对于计算页面加载时间分位数，此函数比 [分位数](#quantile). + 对于计算页面加载时间分位数，此函数比 [分位数](#quantile)更高效和准确。 **返回值** -- 指定电平的分位数。 +- 指定层次的分位数。 类型: `Float32`. !!! note "注" - 如果没有值传递给函数（当使用 `quantileTimingIf`), [阿南](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 被返回。 这样做的目的是将这些案例与导致零的案例区分开来。 看 [按条款订购](../statements/select/order-by.md#select-order-by) 对于排序注意事项 `NaN` 值。 + 如果没有值传递给函数（当使用 `quantileTimingIf`), [NaN](../../sql-reference/data-types/float.md#data_type-float-nan-inf) 被返回。 这样做的目的是将这些案例与导致零的案例区分开来。看 [ORDER BY clause](../statements/select/order-by.md#select-order-by) 对于 `NaN` 值排序注意事项。 **示例** @@ -1384,13 +1382,13 @@ SELECT quantileTimingWeighted(response_time, weight) FROM t ## quantileTDigest {#quantiletdigest} -计算近似值 [分位数](https://en.wikipedia.org/wiki/Quantile) 使用的数字数据序列 [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) 算法。 +使用[t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) 算法计算近似[分位数](https://en.wikipedia.org/wiki/Quantile)。 -最大误差为1%。 内存消耗 `log(n)`，哪里 `n` 是多个值。 结果取决于运行查询的顺序，并且是不确定的。 +最大误差为1%。 内存消耗 `log(n)`，这里 `n` 是值的个数。 结果取决于运行查询的顺序，并且是不确定的。 -该功能的性能低于性能 [分位数](#quantile) 或 [分位定时](#quantiletiming). 在状态大小与精度的比率方面，这个函数比 `quantile`. +该功能的性能低于性能 [分位数](#quantile) 或 [时间分位](#quantiletiming). 在状态大小与精度的比率方面，这个函数比 `quantile`更优秀。 -当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。 +当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能。 **语法** @@ -1402,12 +1400,12 @@ quantileTDigest(level)(expr) **参数** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md). +- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). +- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。 **回值** -- 指定电平的近似分位数。 +- 指定层次的分位数。 类型: @@ -1438,13 +1436,13 @@ SELECT quantileTDigest(number) FROM numbers(10) ## quantileTDigestWeighted {#quantiletdigestweighted} -计算近似值 [分位数](https://en.wikipedia.org/wiki/Quantile) 使用的数字数据序列 [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) 算法。 该函数考虑了每个序列成员的权重。 最大误差为1%。 内存消耗 `log(n)`，哪里 `n` 是多个值。 +使用[t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) 算法计算近似[分位数](https://en.wikipedia.org/wiki/Quantile)。 该函数考虑了每个序列成员的权重。最大误差为1%。 内存消耗 `log(n)`，这里 `n` 是值的个数。 -该功能的性能低于性能 [分位数](#quantile) 或 [分位定时](#quantiletiming). 在状态大小与精度的比率方面，这个函数比 `quantile`. +该功能的性能低于性能 [分位数](#quantile) 或 [时间分位](#quantiletiming). 在状态大小与精度的比率方面，这个函数比 `quantile`更优秀。 结果取决于运行查询的顺序，并且是不确定的。 -当使用多个 `quantile*` 在查询中具有不同级别的函数，内部状态不会被组合（即查询的工作效率低于它可以）。 在这种情况下，使用 [分位数](#quantiles) 功能。 +当在一个查询中使用多个不同层次的 `quantile*` 时，内部状态不会被组合（即查询的工作效率低于组合情况）。在这种情况下，使用[分位数](#quantiles)功能 **语法** @@ -1456,13 +1454,13 @@ quantileTDigest(level)(expr) **参数** -- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` 值的范围 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). -- `expr` — Expression over the column values resulting in numeric [数据类型](../../sql-reference/data-types/index.md#data_types), [日期](../../sql-reference/data-types/date.md) 或 [日期时间](../../sql-reference/data-types/datetime.md). -- `weight` — Column with weights of sequence elements. Weight is a number of value occurrences. +- `level` — 分位数层次。可选参数。 从0到1的一个float类型的常量。 我们推荐 `level` 值的范围为 `[0.01, 0.99]`. 默认值：0.5。 在 `level=0.5` 该函数计算 [中位数](https://en.wikipedia.org/wiki/Median). +- `expr` — 求职表达式，类型为：数值[数据类型](../../sql-reference/data-types/index.md#data_types),[日期](../../sql-reference/data-types/date.md)数据类型或[时间](../../sql-reference/data-types/datetime.md)数据类型。 +- `weight` — 权重序列。 权重是一个数据出现的数值。 **返回值** -- 指定电平的近似分位数。 +- 指定层次的分位数。 类型: @@ -1491,20 +1489,20 @@ SELECT quantileTDigestWeighted(number, 1) FROM numbers(10) - [中位数](#median) - [分位数](#quantiles) -## 中位数 {#median} +## median {#median} -该 `median*` 函数是相应的别名 `quantile*` 功能。 它们计算数字数据样本的中位数。 +`median*` 函数是 `quantile*` 函数的别名。 它们计算数字数据样本的中位数。 -功能: +函数: -- `median` — Alias for [分位数](#quantile). -- `medianDeterministic` — Alias for [量化确定](#quantiledeterministic). -- `medianExact` — Alias for [quantileExact](#quantileexact). -- `medianExactWeighted` — Alias for [分位数加权](#quantileexactweighted). -- `medianTiming` — Alias for [分位定时](#quantiletiming). -- `medianTimingWeighted` — Alias for [分位时间加权](#quantiletimingweighted). -- `medianTDigest` — Alias for [quantileTDigest](#quantiletdigest). -- `medianTDigestWeighted` — Alias for [quantileTDigestWeighted](#quantiletdigestweighted). +- `median` — [quantile](#quantile)别名。 +- `medianDeterministic` — [quantileDeterministic](#quantiledeterministic)别名。 +- `medianExact` — [quantileExact](#quantileexact)别名。 +- `medianExactWeighted` — [quantileExactWeighted](#quantileexactweighted)别名。 +- `medianTiming` — [quantileTiming](#quantiletiming)别名。 +- `medianTimingWeighted` — [quantileTimingWeighted](#quantiletimingweighted)别名。 +- `medianTDigest` — [quantileTDigest](#quantiletdigest)别名。 +- `medianTDigestWeighted` — [quantileTDigestWeighted](#quantiletdigestweighted)别名。 **示例** @@ -1535,11 +1533,11 @@ SELECT medianDeterministic(val, 1) FROM t ## quantiles(level1, level2, …)(x) {#quantiles} -所有分位数函数也具有相应的分位数函数: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. 这些函数在一遍中计算所列电平的所有分位数，并返回结果值的数组。 +所有分位数函数也有相应的函数: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`。这些函数一次计算所列层次的所有分位数，并返回结果值的数组。 ## varSamp(x) {#varsampx} -计算金额 `Σ((x - x̅)^2) / (n - 1)`，哪里 `n` 是样本大小和 `x̅`是平均值 `x`. +计算 `Σ((x - x̅)^2) / (n - 1)`，这里 `n` 是样本大小， `x̅`是`x`的平均值。 它表示随机变量的方差的无偏估计，如果传递的值形成其样本。 @@ -1550,23 +1548,23 @@ SELECT medianDeterministic(val, 1) FROM t ## varPop(x) {#varpopx} -计算金额 `Σ((x - x̅)^2) / n`，哪里 `n` 是样本大小和 `x̅`是平均值 `x`. +计算 `Σ((x - x̅)^2) / n`，这里 `n` 是样本大小， `x̅`是`x`的平均值。 -换句话说，分散为一组值。 返回 `Float64`. +换句话说，计算一组数据的离差。 返回 `Float64`。 !!! note "注" 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `varPopStable` 功能。 它的工作速度较慢，但提供较低的计算错误。 ## stddevSamp(x) {#stddevsampx} -结果等于平方根 `varSamp(x)`. +结果等于平方根 `varSamp(x)`。 !!! note "注" 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `stddevSampStable` 功能。 它的工作速度较慢，但提供较低的计算错误。 ## stddevPop(x) {#stddevpopx} -结果等于平方根 `varPop(x)`. +结果等于平方根 `varPop(x)`。 !!! note "注" 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `stddevPopStable` 功能。 它的工作速度较慢，但提供较低的计算错误。 @@ -1575,15 +1573,15 @@ SELECT medianDeterministic(val, 1) FROM t 返回指定列中近似最常见值的数组。 生成的数组按值的近似频率降序排序（而不是值本身）。 -实现了 [过滤节省空间](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) 基于reduce-and-combine算法的TopK分析算法 [并行节省空间](https://arxiv.org/pdf/1401.0702.pdf). +实现了[过滤节省空间](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf)算法， 使用基于reduce-and-combine的算法，借鉴[并行节省空间](https://arxiv.org/pdf/1401.0702.pdf). ``` sql topK(N)(column) ``` -此函数不提供保证的结果。 在某些情况下，可能会发生错误，并且可能会返回不是最常见值的常见值。 +此函数不提供保证的结果。 在某些情况下，可能会发生错误，并且可能会返回不是最高频的值。 -我们建议使用 `N < 10` 值;性能降低了大 `N` 值。 的最大值 `N = 65536`. +我们建议使用 `N < 10` 值，`N` 值越大，性能越低。最大值 `N = 65536`。 **参数** @@ -1593,11 +1591,11 @@ topK(N)(column) **参数** -- ' x ' – The value to calculate frequency. +- ' x ' – 计算的频率值。 **示例** -就拿 [时间](../../getting-started/example-datasets/ontime.md) 数据集，并选择在三个最频繁出现的值 `AirlineID` 列。 +就拿 [OnTime](../../getting-started/example-datasets/ontime.md) 数据集来说，选择`AirlineID` 列中出现最频繁的三个。 ``` sql SELECT topK(3)(AirlineID) AS res @@ -1612,7 +1610,7 @@ FROM ontime ## topKWeighted {#topkweighted} -类似于 `topK` 但需要一个整数类型的附加参数 - `weight`. 每个价值都被记入 `weight` 次频率计算。 +类似于 `topK` 但需要一个整数类型的附加参数 - `weight`. 每个输入都被记入 `weight` 次频率计算。 **语法** @@ -1622,12 +1620,12 @@ topKWeighted(N)(x, weight) **参数** -- `N` — The number of elements to return. +- `N` — 返回值个数。 **参数** -- `x` – The value. -- `weight` — The weight. [UInt8](../../sql-reference/data-types/int-uint.md). +- `x` – 输入值。 +- `weight` — 权重。 [UInt8](../../sql-reference/data-types/int-uint.md)类型。 **返回值** @@ -1651,36 +1649,36 @@ SELECT topKWeighted(10)(number, number) FROM numbers(1000) ## covarSamp(x,y) {#covarsampx-y} -计算的值 `Σ((x - x̅)(y - y̅)) / (n - 1)`. +计算 `Σ((x - x̅)(y - y̅)) / (n - 1)`。 -返回Float64。 当 `n <= 1`, returns +∞. +返回Float64。 当 `n <= 1`, returns +∞。 !!! note "注" 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `covarSampStable` 功能。 它的工作速度较慢，但提供较低的计算错误。 ## covarPop(x,y) {#covarpopx-y} -计算的值 `Σ((x - x̅)(y - y̅)) / n`. +计算 `Σ((x - x̅)(y - y̅)) / n`。 !!! note "注" 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `covarPopStable` 功能。 它的工作速度较慢，但提供了较低的计算错误。 ## corr(x,y) {#corrx-y} -计算Pearson相关系数: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`. +计算Pearson相关系数: `Σ((x - x̅)(y - y̅)) / sqrt(Σ((x - x̅)^2) * Σ((y - y̅)^2))`。 !!! note "注" 该函数使用数值不稳定的算法。 如果你需要 [数值稳定性](https://en.wikipedia.org/wiki/Numerical_stability) 在计算中，使用 `corrStable` 功能。 它的工作速度较慢，但提供较低的计算错误。 ## categoricalInformationValue {#categoricalinformationvalue} -计算的值 `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` 对于每个类别。 +对于每个类别计算 `(P(tag = 1) - P(tag = 0))(log(P(tag = 1)) - log(P(tag = 0)))` 。 ``` sql categoricalInformationValue(category1, category2, ..., tag) ``` -结果指示离散（分类）要素如何使用 `[category1, category2, ...]` 有助于预测的价值的学习模型 `tag`. +结果指示离散（分类）要素如何使用 `[category1, category2, ...]` 有助于使用学习模型预测`tag`的值。 ## simpleLinearRegression {#simplelinearregression} @@ -1692,12 +1690,12 @@ simpleLinearRegression(x, y) 参数: -- `x` — Column with dependent variable values. -- `y` — Column with explanatory variable values. +- `x` — x轴。 +- `y` — y轴。 返回值: -常量 `(a, b)` 结果行的 `y = a*x + b`. +符合`y = a*x + b`的常量 `(a, b)` 。 **例** @@ -1721,9 +1719,9 @@ SELECT arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [3, 4, 5, 6]) └───────────────────────────────────────────────────────────────────┘ ``` -## 随机指标线上回归 {#agg_functions-stochasticlinearregression} +## stochasticLinearRegression {#agg_functions-stochasticlinearregression} -该函数实现随机线性回归。 它支持自定义参数的学习率，L2正则化系数，迷你批量大小，并具有更新权重的方法很少 ([亚当](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) （默认使用), [简单SGD](https://en.wikipedia.org/wiki/Stochastic_gradient_descent), [动量](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum), [Nesterov](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)). +该函数实现随机线性回归。 它支持自定义参数的学习率、L2正则化系数、微批，并且具有少量更新权重的方法（[Adam](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Adam) （默认）， [simple SGD](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)， [Momentum](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#Momentum)， [Nesterov](https://mipt.ru/upload/medialibrary/d7e/41-91.pdf)）。 ### 参数 {#agg_functions-stochasticlinearregression-parameters} @@ -1738,14 +1736,14 @@ stochasticLinearRegression(1.0, 1.0, 10, 'SGD') 3. `mini-batch size` 设置元素的数量，这些元素将被计算和求和以执行梯度下降的一个步骤。 纯随机下降使用一个元素，但是具有小批量（约10个元素）使梯度步骤更稳定。 默认值为 `15`. 4. `method for updating weights` 他们是: `Adam` （默认情况下), `SGD`, `Momentum`, `Nesterov`. `Momentum` 和 `Nesterov` 需要更多的计算和内存，但是它们恰好在收敛速度和随机梯度方法的稳定性方面是有用的。 -### 用途 {#agg_functions-stochasticlinearregression-usage} +### 用法 {#agg_functions-stochasticlinearregression-usage} `stochasticLinearRegression` 用于两个步骤：拟合模型和预测新数据。 为了拟合模型并保存其状态以供以后使用，我们使用 `-State` combinator，它基本上保存了状态（模型权重等）。 为了预测我们使用函数 [evalMLMethod](../functions/machine-learning-functions.md#machine_learning_methods-evalmlmethod)，这需要一个状态作为参数以及特征来预测。 -**1.** 适合 +**1.** 安装 可以使用这种查询。 @@ -1807,28 +1805,28 @@ evalMLMethod(model, param1, param2) FROM test_data stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') ``` -1. 适合 +**1.** 安装 - See the `Fitting` section in the [stochasticLinearRegression](#stochasticlinearregression-usage-fitting) description. + 参考stochasticLinearRegression相关文档 - Predicted labels have to be in \[-1, 1\]. + 预测标签的取值范围为[-1, 1] -1. 预测 +**2.** 预测 - Using saved state we can predict probability of object having label `1`. + 使用已经保存的state我们可以预测标签为 `1` 的对象的概率。 ``` sql WITH (SELECT state FROM your_model) AS model SELECT evalMLMethod(model, param1, param2) FROM test_data ``` - The query will return a column of probabilities. Note that first argument of `evalMLMethod` is `AggregateFunctionState` object, next are columns of features. + 查询结果返回一个列的概率。注意 `evalMLMethod` 的第一个参数是 `AggregateFunctionState` 对象，接下来的参数是列的特性。 - We can also set a bound of probability, which assigns elements to different labels. + 我们也可以设置概率的范围， 这样需要给元素指定不同的标签。 ``` sql SELECT ans < 1.1 AND ans > 0.5 FROM @@ -1836,14 +1834,14 @@ stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') evalMLMethod(model, param1, param2) AS ans FROM test_data) ``` - Then the result will be labels. + 结果是标签。 - `test_data` is a table like `train_data` but may not contain target value. + `test_data` 是一个像 `train_data` 一样的表，但是不包含目标值。 **另请参阅** - [随机指标线上回归](#agg_functions-stochasticlinearregression) -- [线性回归和逻辑回归之间的差异。](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) +- [线性回归和逻辑回归之间的差异](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) ## groupBitmapAnd {#groupbitmapand} From 2c439afc015b15d9babb632b423991aeea0f397f Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Fri, 12 Jun 2020 19:17:34 +0800 Subject: [PATCH 106/329] ISSUES-7572 fix build failure --- src/Server/ReplicasStatusHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 9b3e00cc069..d6bbfdbd090 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -110,7 +110,7 @@ void addReplicasStatusHandlerFactory(HTTPRequestHandlerFactoryMain & factory, IS { auto replicas_status_handler = std::make_unique>(server); replicas_status_handler->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest(); - factory->addHandler(replicas_status_handler.release()); + factory.addHandler(replicas_status_handler.release()); } } From 787163d0b489c464d33b3ba9ee003dcdaa4bf6a9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 17:03:00 +0300 Subject: [PATCH 107/329] Minor modifications after merging #11554 --- src/Functions/extractAllGroups.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/extractAllGroups.h b/src/Functions/extractAllGroups.h index a9206e7327e..b75e54b490e 100644 --- a/src/Functions/extractAllGroups.h +++ b/src/Functions/extractAllGroups.h @@ -54,7 +54,7 @@ public: size_t getNumberOfArguments() const override { return 2; } - bool useDefaultImplementationForConstants() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override From 0edf5ff7a25fea2bc9028a02ec0a147b0ca47f4d Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 12 Jun 2020 17:32:47 +0300 Subject: [PATCH 108/329] Fix race condition --- src/Interpreters/ExpressionActions.h | 2 + src/Storages/IStorage.cpp | 85 ++++++++++++++++++------ src/Storages/IStorage.h | 37 ++++++----- src/Storages/IndicesDescription.cpp | 10 ++- src/Storages/KeyDescription.cpp | 13 +++- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/TTLDescription.cpp | 47 +++++++++++-- src/Storages/TTLDescription.h | 4 ++ 8 files changed, 153 insertions(+), 47 deletions(-) diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 080e8f8a10f..26493f857b0 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -163,6 +163,8 @@ public: ~ExpressionActions(); + ExpressionActions(const ExpressionActions & other) = default; + /// Add the input column. /// The name of the column must not match the names of the intermediate columns that occur when evaluating the expression. /// The expression must not have any PROJECT actions. diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 58cbb2bb7d6..a09bb45f9d0 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -32,24 +32,28 @@ namespace ErrorCodes extern const int DEADLOCK_AVOIDED; } -const ColumnsDescription & IStorage::getColumns() const +ColumnsDescription IStorage::getColumns() const { + std::lock_guard lock(metadata_mutex); return metadata.columns; } -const IndicesDescription & IStorage::getSecondaryIndices() const +IndicesDescription IStorage::getSecondaryIndices() const { + std::lock_guard lock(metadata_mutex); return metadata.secondary_indices; } bool IStorage::hasSecondaryIndices() const { + std::lock_guard lock(metadata_mutex); return !metadata.secondary_indices.empty(); } -const ConstraintsDescription & IStorage::getConstraints() const +ConstraintsDescription IStorage::getConstraints() const { + std::lock_guard lock(metadata_mutex); return metadata.constraints; } @@ -290,6 +294,7 @@ void IStorage::check(const Block & block, bool need_all) const void IStorage::setColumns(ColumnsDescription columns_) { + std::lock_guard lock(metadata_mutex); if (columns_.getOrdinary().empty()) throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); metadata.columns = std::move(columns_); @@ -297,11 +302,13 @@ void IStorage::setColumns(ColumnsDescription columns_) void IStorage::setSecondaryIndices(IndicesDescription secondary_indices_) { + std::lock_guard lock(metadata_mutex); metadata.secondary_indices = std::move(secondary_indices_); } void IStorage::setConstraints(ConstraintsDescription constraints_) { + std::lock_guard lock(metadata_mutex); metadata.constraints = std::move(constraints_); } @@ -416,136 +423,160 @@ NamesAndTypesList IStorage::getVirtuals() const return {}; } -const KeyDescription & IStorage::getPartitionKey() const +KeyDescription IStorage::getPartitionKey() const { + std::lock_guard lock(metadata_mutex); return metadata.partition_key; } void IStorage::setPartitionKey(const KeyDescription & partition_key_) { + std::lock_guard lock(metadata_mutex); metadata.partition_key = partition_key_; } bool IStorage::isPartitionKeyDefined() const { + std::lock_guard lock(metadata_mutex); return metadata.partition_key.definition_ast != nullptr; } bool IStorage::hasPartitionKey() const { + std::lock_guard lock(metadata_mutex); return !metadata.partition_key.column_names.empty(); } Names IStorage::getColumnsRequiredForPartitionKey() const { - if (hasPartitionKey()) + std::lock_guard lock(metadata_mutex); + if (!metadata.partition_key.column_names.empty()) return metadata.partition_key.expression->getRequiredColumns(); return {}; } -const KeyDescription & IStorage::getSortingKey() const +KeyDescription IStorage::getSortingKey() const { + std::lock_guard lock(metadata_mutex); return metadata.sorting_key; } void IStorage::setSortingKey(const KeyDescription & sorting_key_) { + std::lock_guard lock(metadata_mutex); metadata.sorting_key = sorting_key_; } bool IStorage::isSortingKeyDefined() const { + std::lock_guard lock(metadata_mutex); return metadata.sorting_key.definition_ast != nullptr; } bool IStorage::hasSortingKey() const { + std::lock_guard lock(metadata_mutex); return !metadata.sorting_key.column_names.empty(); } Names IStorage::getColumnsRequiredForSortingKey() const { - if (hasSortingKey()) + std::lock_guard lock(metadata_mutex); + if (!metadata.sorting_key.column_names.empty()) return metadata.sorting_key.expression->getRequiredColumns(); return {}; } Names IStorage::getSortingKeyColumns() const { - if (hasSortingKey()) + std::lock_guard lock(metadata_mutex); + if (!metadata.sorting_key.column_names.empty()) return metadata.sorting_key.column_names; return {}; } -const KeyDescription & IStorage::getPrimaryKey() const +KeyDescription IStorage::getPrimaryKey() const { + std::lock_guard lock(metadata_mutex); return metadata.primary_key; } void IStorage::setPrimaryKey(const KeyDescription & primary_key_) { + std::lock_guard lock(metadata_mutex); metadata.primary_key = primary_key_; } bool IStorage::isPrimaryKeyDefined() const { + std::lock_guard lock(metadata_mutex); return metadata.primary_key.definition_ast != nullptr; } bool IStorage::hasPrimaryKey() const { + std::lock_guard lock(metadata_mutex); return !metadata.primary_key.column_names.empty(); } Names IStorage::getColumnsRequiredForPrimaryKey() const { - if (hasPrimaryKey()) + std::lock_guard lock(metadata_mutex); + if (!metadata.primary_key.column_names.empty()) return metadata.primary_key.expression->getRequiredColumns(); return {}; } Names IStorage::getPrimaryKeyColumns() const { - if (hasSortingKey()) + std::lock_guard lock(metadata_mutex); + if (!metadata.primary_key.column_names.empty()) return metadata.primary_key.column_names; return {}; } -const KeyDescription & IStorage::getSamplingKey() const +KeyDescription IStorage::getSamplingKey() const { + std::lock_guard lock(metadata_mutex); return metadata.sampling_key; } void IStorage::setSamplingKey(const KeyDescription & sampling_key_) { + std::lock_guard lock(metadata_mutex); metadata.sampling_key = sampling_key_; } bool IStorage::isSamplingKeyDefined() const { + std::lock_guard lock(metadata_mutex); return metadata.sampling_key.definition_ast != nullptr; } bool IStorage::hasSamplingKey() const { + std::lock_guard lock(metadata_mutex); return !metadata.sampling_key.column_names.empty(); } Names IStorage::getColumnsRequiredForSampling() const { - if (hasSamplingKey()) + std::lock_guard lock(metadata_mutex); + if (!metadata.sampling_key.column_names.empty()) return metadata.sampling_key.expression->getRequiredColumns(); return {}; } -const TTLTableDescription & IStorage::getTableTTLs() const +TTLTableDescription IStorage::getTableTTLs() const { + std::lock_guard lock(metadata_mutex); return metadata.table_ttl; } void IStorage::setTableTTLs(const TTLTableDescription & table_ttl_) { + std::lock_guard lock(metadata_mutex); metadata.table_ttl = table_ttl_; } @@ -554,38 +585,45 @@ bool IStorage::hasAnyTableTTL() const return hasAnyMoveTTL() || hasRowsTTL(); } -const TTLColumnsDescription & IStorage::getColumnTTLs() const +TTLColumnsDescription IStorage::getColumnTTLs() const { + std::lock_guard lock(metadata_mutex); return metadata.column_ttls_by_name; } void IStorage::setColumnTTLs(const TTLColumnsDescription & column_ttls_by_name_) { + std::lock_guard lock(metadata_mutex); metadata.column_ttls_by_name = column_ttls_by_name_; } bool IStorage::hasAnyColumnTTL() const { + std::lock_guard lock(metadata_mutex); return !metadata.column_ttls_by_name.empty(); } -const TTLDescription & IStorage::getRowsTTL() const +TTLDescription IStorage::getRowsTTL() const { + std::lock_guard lock(metadata_mutex); return metadata.table_ttl.rows_ttl; } bool IStorage::hasRowsTTL() const { + std::lock_guard lock(metadata_mutex); return metadata.table_ttl.rows_ttl.expression != nullptr; } -const TTLDescriptions & IStorage::getMoveTTLs() const +TTLDescriptions IStorage::getMoveTTLs() const { + std::lock_guard lock(metadata_mutex); return metadata.table_ttl.move_ttl; } bool IStorage::hasAnyMoveTTL() const { + std::lock_guard lock(metadata_mutex); return !metadata.table_ttl.move_ttl.empty(); } @@ -649,31 +687,38 @@ ColumnDependencies IStorage::getColumnDependencies(const NameSet & updated_colum } -const ASTPtr & IStorage::getSettingsChanges() const +ASTPtr IStorage::getSettingsChanges() const { - return metadata.settings_changes; + std::lock_guard lock(metadata_mutex); + if (metadata.settings_changes) + return metadata.settings_changes->clone(); + return nullptr; } void IStorage::setSettingsChanges(const ASTPtr & settings_changes_) { + std::lock_guard lock(metadata_mutex); if (settings_changes_) metadata.settings_changes = settings_changes_->clone(); else metadata.settings_changes = nullptr; } -const SelectQueryDescription & IStorage::getSelectQuery() const +SelectQueryDescription IStorage::getSelectQuery() const { + std::lock_guard lock(metadata_mutex); return metadata.select; } void IStorage::setSelectQuery(const SelectQueryDescription & select_) { + std::lock_guard lock(metadata_mutex); metadata.select = select_; } bool IStorage::hasSelectQuery() const { + std::lock_guard lock(metadata_mutex); return metadata.select.select_query != nullptr; } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 1a5a0a0753d..40ca901640b 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -137,29 +137,33 @@ public: using ColumnSizeByName = std::unordered_map; virtual ColumnSizeByName getColumnSizes() const { return {}; } -public: /// thread-unsafe part. lockStructure must be acquired - const ColumnsDescription & getColumns() const; /// returns combined set of columns +public: + /// NOTE: These methods are thread-safe now, but require additional + /// structure lock to get consistent metadata snapshot. This will be fixed + /// soon. TODO(alesap) + + ColumnsDescription getColumns() const; /// returns combined set of columns void setColumns(ColumnsDescription columns_); /// sets only real columns, possibly overwrites virtual ones. void setSecondaryIndices(IndicesDescription secondary_indices_); - const IndicesDescription & getSecondaryIndices() const; + IndicesDescription getSecondaryIndices() const; /// Has at least one non primary index bool hasSecondaryIndices() const; - const ConstraintsDescription & getConstraints() const; + ConstraintsDescription getConstraints() const; void setConstraints(ConstraintsDescription constraints_); /// Storage settings - const ASTPtr & getSettingsChanges() const; + ASTPtr getSettingsChanges() const; void setSettingsChanges(const ASTPtr & settings_changes_); bool hasSettingsChanges() const { return metadata.settings_changes != nullptr; } /// Select query for *View storages. - const SelectQueryDescription & getSelectQuery() const; + SelectQueryDescription getSelectQuery() const; void setSelectQuery(const SelectQueryDescription & select_); bool hasSelectQuery() const; - const StorageInMemoryMetadata & getInMemoryMetadata() const { return metadata; } + StorageInMemoryMetadata getInMemoryMetadata() const { return metadata; } Block getSampleBlock() const; /// ordinary + materialized. Block getSampleBlockWithVirtuals() const; /// ordinary + materialized + virtuals. @@ -204,7 +208,8 @@ private: StorageID storage_id; mutable std::mutex id_mutex; - + /// TODO (alesap) just use multiversion for atomic metadata + mutable std::mutex metadata_mutex; StorageInMemoryMetadata metadata; private: RWLockImpl::LockHolder tryLockTimed( @@ -438,7 +443,7 @@ public: virtual Strings getDataPaths() const { return {}; } /// Returns structure with partition key. - const KeyDescription & getPartitionKey() const; + KeyDescription getPartitionKey() const; /// Set partition key for storage (methods bellow, are just wrappers for this /// struct). void setPartitionKey(const KeyDescription & partition_key_); @@ -453,7 +458,7 @@ public: /// Returns structure with sorting key. - const KeyDescription & getSortingKey() const; + KeyDescription getSortingKey() const; /// Set sorting key for storage (methods bellow, are just wrappers for this /// struct). void setSortingKey(const KeyDescription & sorting_key_); @@ -470,7 +475,7 @@ public: Names getSortingKeyColumns() const; /// Returns structure with primary key. - const KeyDescription & getPrimaryKey() const; + KeyDescription getPrimaryKey() const; /// Set primary key for storage (methods bellow, are just wrappers for this /// struct). void setPrimaryKey(const KeyDescription & primary_key_); @@ -488,7 +493,7 @@ public: Names getPrimaryKeyColumns() const; /// Returns structure with sampling key. - const KeyDescription & getSamplingKey() const; + KeyDescription getSamplingKey() const; /// Set sampling key for storage (methods bellow, are just wrappers for this /// struct). void setSamplingKey(const KeyDescription & sampling_key_); @@ -512,22 +517,22 @@ public: virtual StoragePolicyPtr getStoragePolicy() const { return {}; } /// Common tables TTLs (for rows and moves). - const TTLTableDescription & getTableTTLs() const; + TTLTableDescription getTableTTLs() const; void setTableTTLs(const TTLTableDescription & table_ttl_); bool hasAnyTableTTL() const; /// Separate TTLs for columns. - const TTLColumnsDescription & getColumnTTLs() const; + TTLColumnsDescription getColumnTTLs() const; void setColumnTTLs(const TTLColumnsDescription & column_ttls_by_name_); bool hasAnyColumnTTL() const; /// Just wrapper for table TTLs, return rows part of table TTLs. - const TTLDescription & getRowsTTL() const; + TTLDescription getRowsTTL() const; bool hasRowsTTL() const; /// Just wrapper for table TTLs, return moves (to disks or volumes) parts of /// table TTL. - const TTLDescriptions & getMoveTTLs() const; + TTLDescriptions getMoveTTLs() const; bool hasAnyMoveTTL() const; /// If it is possible to quickly determine exact number of rows in the table at this moment of time, then return it. diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index d59aef2ecaa..ee9a9681e61 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -24,13 +24,14 @@ IndexDescription::IndexDescription(const IndexDescription & other) , expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr) , name(other.name) , type(other.type) - , expression(other.expression) /// actions never changed , arguments(other.arguments) , column_names(other.column_names) , data_types(other.data_types) , sample_block(other.sample_block) , granularity(other.granularity) { + if (other.expression) + expression = std::make_shared(*other.expression); } @@ -51,7 +52,12 @@ IndexDescription & IndexDescription::operator=(const IndexDescription & other) name = other.name; type = other.type; - expression = other.expression; + + if (other.expression) + expression = std::make_shared(*other.expression); + else + expression.reset(); + arguments = other.arguments; column_names = other.column_names; data_types = other.data_types; diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index 4eb18320ad9..7d5b0d56008 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -19,12 +19,13 @@ namespace ErrorCodes KeyDescription::KeyDescription(const KeyDescription & other) : definition_ast(other.definition_ast ? other.definition_ast->clone() : nullptr) , expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr) - , expression(other.expression) , sample_block(other.sample_block) , column_names(other.column_names) , data_types(other.data_types) , additional_column(other.additional_column) { + if (other.expression) + expression = std::make_shared(*other.expression); } KeyDescription & KeyDescription::operator=(const KeyDescription & other) @@ -39,7 +40,15 @@ KeyDescription & KeyDescription::operator=(const KeyDescription & other) if (other.expression_list_ast) expression_list_ast = other.expression_list_ast->clone(); - expression = other.expression; + else + expression_list_ast.reset(); + + + if (other.expression) + expression = std::make_shared(*other.expression); + else + expression.reset(); + sample_block = other.sample_block; column_names = other.column_names; data_types = other.data_types; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 7f8de6f3856..fdd62b03046 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1344,7 +1344,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S if (hasSettingsChanges()) { - const auto & current_changes = getSettingsChanges()->as().changes; + const auto current_changes = getSettingsChanges()->as().changes; const auto & new_changes = new_metadata.settings_changes->as().changes; for (const auto & changed_setting : new_changes) { diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index e241b7676a0..898df5006fd 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -23,6 +23,26 @@ extern const int BAD_ARGUMENTS; extern const int BAD_TTL_EXPRESSION; } + +TTLAggregateDescription::TTLAggregateDescription(const TTLAggregateDescription & other) + : column_name(other.column_name) + , expression_result_column_name(other.expression_result_column_name) +{ + if (other.expression) + expression = std::make_shared(*other.expression); +} + +TTLAggregateDescription & TTLAggregateDescription::operator=(const TTLAggregateDescription & other) +{ + column_name = other.column_name; + expression_result_column_name = other.expression_result_column_name; + if (other.expression) + expression = std::make_shared(*other.expression); + else + expression.reset(); + return *this; +} + namespace { @@ -58,9 +78,7 @@ void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const Strin TTLDescription::TTLDescription(const TTLDescription & other) : mode(other.mode) , expression_ast(other.expression_ast ? other.expression_ast->clone() : nullptr) - , expression(other.expression) , result_column(other.result_column) - , where_expression(other.where_expression) , where_result_column(other.where_result_column) , group_by_keys(other.group_by_keys) , set_parts(other.set_parts) @@ -68,6 +86,11 @@ TTLDescription::TTLDescription(const TTLDescription & other) , destination_type(other.destination_type) , destination_name(other.destination_name) { + if (other.expression) + expression = std::make_shared(*other.expression); + + if (other.where_expression) + where_expression = std::make_shared(*other.where_expression); } TTLDescription & TTLDescription::operator=(const TTLDescription & other) @@ -81,9 +104,17 @@ TTLDescription & TTLDescription::operator=(const TTLDescription & other) else expression_ast.reset(); - expression = other.expression; + if (other.expression) + expression = std::make_shared(*other.expression); + else + expression.reset(); + result_column = other.result_column; - where_expression = other.where_expression; + if (other.where_expression) + where_expression = std::make_shared(*other.where_expression); + else + where_expression.reset(); + where_result_column = other.where_result_column; group_by_keys = other.group_by_keys; set_parts = other.set_parts; @@ -218,8 +249,12 @@ TTLDescription TTLDescription::getTTLFromAST( auto syntax_result = SyntaxAnalyzer(context).analyze(value, columns.getAllPhysical(), {}, true); auto expr_analyzer = ExpressionAnalyzer(value, syntax_result, context); - result.set_parts.emplace_back(TTLAggregateDescription{ - name, value->getColumnName(), expr_analyzer.getActions(false)}); + TTLAggregateDescription set_part; + set_part.column_name = name; + set_part.expression_result_column_name = value->getColumnName(); + set_part.expression = expr_analyzer.getActions(false); + + result.set_parts.emplace_back(set_part); for (const auto & descr : expr_analyzer.getAnalyzedData().aggregate_descriptions) result.aggregate_descriptions.push_back(descr); diff --git a/src/Storages/TTLDescription.h b/src/Storages/TTLDescription.h index 86e82e14c73..906cfb0e675 100644 --- a/src/Storages/TTLDescription.h +++ b/src/Storages/TTLDescription.h @@ -25,6 +25,10 @@ struct TTLAggregateDescription /// Expressions to calculate the value of assignment expression ExpressionActionsPtr expression; + + TTLAggregateDescription() = default; + TTLAggregateDescription(const TTLAggregateDescription & other); + TTLAggregateDescription & operator=(const TTLAggregateDescription & other); }; using TTLAggregateDescriptions = std::vector; From 83155e139c681836adc4a843d88a8f6839981f88 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 12 Jun 2020 17:59:14 +0300 Subject: [PATCH 109/329] Try fix tests. --- src/Storages/MergeTree/MergeTreeRangeReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 6cdd0270dea..667b0b2da96 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -906,7 +906,7 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r result.columns.erase(result.columns.begin() + prewhere_column_pos); else result.columns[prewhere_column_pos] = - result.block_before_prewhere.getByPosition(prewhere_column_pos).type-> + getSampleBlock().getByName(prewhere->prewhere_column_name).type-> createColumnConst(result.num_rows, 1u)->convertToFullColumnIfConst(); } } From 72257061d5678606f75faa33b0ca72d54ae1fec8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 18:09:12 +0300 Subject: [PATCH 110/329] Avoid errors due to implicit int<->bool conversions when using ZK API --- programs/copier/ClusterCopier.cpp | 32 +-- programs/copier/ZooKeeperStaff.h | 2 +- src/Common/ZooKeeper/IKeeper.cpp | 94 ++++---- src/Common/ZooKeeper/IKeeper.h | 113 +++++----- src/Common/ZooKeeper/Increment.h | 4 +- src/Common/ZooKeeper/KeeperException.h | 6 +- src/Common/ZooKeeper/LeaderElection.h | 2 +- src/Common/ZooKeeper/Lock.cpp | 6 +- src/Common/ZooKeeper/TestKeeper.cpp | 14 +- src/Common/ZooKeeper/ZooKeeper.cpp | 207 +++++++++--------- src/Common/ZooKeeper/ZooKeeper.h | 40 ++-- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 103 +++++---- .../gtest_zkutil_test_multi_exception.cpp | 8 +- .../tests/zkutil_expiration_test.cpp | 6 +- .../tests/zkutil_test_commands_new_lib.cpp | 44 ++-- src/Common/ZooKeeper/tests/zookeeper_impl.cpp | 4 +- src/Interpreters/DDLWorker.cpp | 19 +- .../MergeTree/EphemeralLockInZooKeeper.cpp | 6 +- .../ReplicatedMergeTreeBlockOutputStream.cpp | 22 +- .../ReplicatedMergeTreeCleanupThread.cpp | 14 +- .../ReplicatedMergeTreePartCheckThread.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 18 +- .../ReplicatedMergeTreeRestartingThread.cpp | 8 +- src/Storages/StorageReplicatedMergeTree.cpp | 122 +++++------ .../System/StorageSystemZooKeeper.cpp | 2 +- .../get_current_inserts_in_replicated.cpp | 2 +- .../tests/transform_part_zk_nodes.cpp | 2 +- utils/zookeeper-dump-tree/main.cpp | 2 +- 28 files changed, 462 insertions(+), 442 deletions(-) diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 5254d2a97ac..7fa0f663295 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -25,7 +25,7 @@ void ClusterCopier::init() task_description_watch_callback = [this] (const Coordination::WatchResponse & response) { - if (response.error != Coordination::ZOK) + if (response.error != Coordination::Error::ZOK) return; UInt64 version = ++task_description_version; LOG_DEBUG(log, "Task description should be updated, local version {}", version); @@ -206,11 +206,11 @@ void ClusterCopier::uploadTaskDescription(const std::string & task_path, const s zookeeper->createAncestors(local_task_description_path); auto code = zookeeper->tryCreate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent); - if (code && force) + if (code != Coordination::Error::ZOK && force) zookeeper->createOrUpdate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent); LOG_DEBUG(log, "Task description {} uploaded to {} with result {} ({})", - ((code && !force) ? "not " : ""), local_task_description_path, code, zookeeper->error2string(code)); + ((code != Coordination::Error::ZOK && !force) ? "not " : ""), local_task_description_path, code, Coordination::errorMessage(code)); } void ClusterCopier::reloadTaskDescription() @@ -220,10 +220,10 @@ void ClusterCopier::reloadTaskDescription() String task_config_str; Coordination::Stat stat{}; - int code; + Coordination::Error code; zookeeper->tryGetWatch(task_description_path, task_config_str, &stat, task_description_watch_callback, &code); - if (code) + if (code != Coordination::Error::ZOK) throw Exception("Can't get description node " + task_description_path, ErrorCodes::BAD_ARGUMENTS); LOG_DEBUG(log, "Loading description, zxid={}", task_description_current_stat.czxid); @@ -376,10 +376,10 @@ zkutil::EphemeralNodeHolder::Ptr ClusterCopier::createTaskWorkerNodeAndWaitIfNee Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); - if (code == Coordination::ZOK || code == Coordination::ZNODEEXISTS) + if (code == Coordination::Error::ZOK || code == Coordination::Error::ZNODEEXISTS) return std::make_shared(current_worker_path, *zookeeper, false, false, description); - if (code == Coordination::ZBADVERSION) + if (code == Coordination::Error::ZBADVERSION) { ++num_bad_version_errors; @@ -545,7 +545,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t } catch (const Coordination::Exception & e) { - if (e.code == Coordination::ZNODEEXISTS) + if (e.code == Coordination::Error::ZNODEEXISTS) { LOG_DEBUG(log, "Someone is already moving pieces {}", current_partition_attach_is_active); return TaskStatus::Active; @@ -745,7 +745,7 @@ bool ClusterCopier::tryDropPartitionPiece( } catch (const Coordination::Exception & e) { - if (e.code == Coordination::ZNODEEXISTS) + if (e.code == Coordination::Error::ZNODEEXISTS) { LOG_DEBUG(log, "Partition {} piece {} is cleaning now by somebody, sleep", task_partition.name, toString(current_piece_number)); std::this_thread::sleep_for(default_sleep_time); @@ -778,7 +778,7 @@ bool ClusterCopier::tryDropPartitionPiece( } catch (const Coordination::Exception & e) { - if (e.code == Coordination::ZNODEEXISTS) + if (e.code == Coordination::Error::ZNODEEXISTS) { LOG_DEBUG(log, "Partition {} is being filled now by somebody, sleep", task_partition.name); return false; @@ -795,7 +795,7 @@ bool ClusterCopier::tryDropPartitionPiece( /// Remove all status nodes { Strings children; - if (zookeeper->tryGetChildren(current_shards_path, children) == Coordination::ZOK) + if (zookeeper->tryGetChildren(current_shards_path, children) == Coordination::Error::ZOK) for (const auto & child : children) { zookeeper->removeRecursive(current_shards_path + "/" + child); @@ -845,7 +845,7 @@ bool ClusterCopier::tryDropPartitionPiece( } LOG_INFO(log, "Partition {} piece {} was dropped on cluster {}", task_partition.name, toString(current_piece_number), task_table.cluster_push_name); - if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::ZNODEEXISTS) + if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::Error::ZNODEEXISTS) zookeeper->set(current_shards_path, host_id); } @@ -1233,7 +1233,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( } catch (const Coordination::Exception & e) { - if (e.code == Coordination::ZNODEEXISTS) + if (e.code == Coordination::Error::ZNODEEXISTS) { LOG_DEBUG(log, "Someone is already processing {}", current_task_piece_is_active_path); return TaskStatus::Active; @@ -1271,9 +1271,9 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( { String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id); auto res = zookeeper->tryCreate(current_task_piece_status_path, state_finished, zkutil::CreateMode::Persistent); - if (res == Coordination::ZNODEEXISTS) + if (res == Coordination::Error::ZNODEEXISTS) LOG_DEBUG(log, "Partition {} piece {} is absent on current replica of a shard. But other replicas have already marked it as done.", task_partition.name, current_piece_number); - if (res == Coordination::ZOK) + if (res == Coordination::Error::ZOK) LOG_DEBUG(log, "Partition {} piece {} is absent on current replica of a shard. Will mark it as done. Other replicas will do the same.", task_partition.name, current_piece_number); return TaskStatus::Finished; } @@ -1429,7 +1429,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( { Coordination::ExistsResponse status = future_is_dirty_checker.get(); - if (status.error != Coordination::ZNONODE) + if (status.error != Coordination::Error::ZNONODE) { LogicalClock dirt_discovery_epoch (status.stat.mzxid); if (dirt_discovery_epoch == clean_state_clock.discovery_zxid) diff --git a/programs/copier/ZooKeeperStaff.h b/programs/copier/ZooKeeperStaff.h index edd0d9e43d2..66036ae2f27 100644 --- a/programs/copier/ZooKeeperStaff.h +++ b/programs/copier/ZooKeeperStaff.h @@ -178,7 +178,7 @@ public: [stale = stale] (const Coordination::WatchResponse & rsp) { auto logger = &Poco::Logger::get("ClusterCopier"); - if (rsp.error == Coordination::ZOK) + if (rsp.error == Coordination::Error::ZOK) { switch (rsp.type) { diff --git a/src/Common/ZooKeeper/IKeeper.cpp b/src/Common/ZooKeeper/IKeeper.cpp index 5c27971038f..cb378ba1e13 100644 --- a/src/Common/ZooKeeper/IKeeper.cpp +++ b/src/Common/ZooKeeper/IKeeper.cpp @@ -23,7 +23,7 @@ namespace ProfileEvents namespace Coordination { -Exception::Exception(const std::string & msg, const int32_t code_, int) +Exception::Exception(const std::string & msg, const Error code_, int) : DB::Exception(msg, DB::ErrorCodes::KEEPER_EXCEPTION), code(code_) { if (Coordination::isUserError(code)) @@ -34,17 +34,17 @@ Exception::Exception(const std::string & msg, const int32_t code_, int) ProfileEvents::increment(ProfileEvents::ZooKeeperOtherExceptions); } -Exception::Exception(const std::string & msg, const int32_t code_) +Exception::Exception(const std::string & msg, const Error code_) : Exception(msg + " (" + errorMessage(code_) + ")", code_, 0) { } -Exception::Exception(const int32_t code_) +Exception::Exception(const Error code_) : Exception(errorMessage(code_), code_, 0) { } -Exception::Exception(const int32_t code_, const std::string & path) +Exception::Exception(const Error code_, const std::string & path) : Exception(std::string{errorMessage(code_)} + ", path: " + path, code_, 0) { } @@ -58,10 +58,10 @@ using namespace DB; static void addRootPath(String & path, const String & root_path) { if (path.empty()) - throw Exception("Path cannot be empty", ZBADARGUMENTS); + throw Exception("Path cannot be empty", Error::ZBADARGUMENTS); if (path[0] != '/') - throw Exception("Path must begin with /", ZBADARGUMENTS); + throw Exception("Path must begin with /", Error::ZBADARGUMENTS); if (root_path.empty()) return; @@ -78,64 +78,62 @@ static void removeRootPath(String & path, const String & root_path) return; if (path.size() <= root_path.size()) - throw Exception("Received path is not longer than root_path", ZDATAINCONSISTENCY); + throw Exception("Received path is not longer than root_path", Error::ZDATAINCONSISTENCY); path = path.substr(root_path.size()); } -const char * errorMessage(int32_t code) +const char * errorMessage(Error code) { switch (code) { - case ZOK: return "Ok"; - case ZSYSTEMERROR: return "System error"; - case ZRUNTIMEINCONSISTENCY: return "Run time inconsistency"; - case ZDATAINCONSISTENCY: return "Data inconsistency"; - case ZCONNECTIONLOSS: return "Connection loss"; - case ZMARSHALLINGERROR: return "Marshalling error"; - case ZUNIMPLEMENTED: return "Unimplemented"; - case ZOPERATIONTIMEOUT: return "Operation timeout"; - case ZBADARGUMENTS: return "Bad arguments"; - case ZINVALIDSTATE: return "Invalid zhandle state"; - case ZAPIERROR: return "API error"; - case ZNONODE: return "No node"; - case ZNOAUTH: return "Not authenticated"; - case ZBADVERSION: return "Bad version"; - case ZNOCHILDRENFOREPHEMERALS: return "No children for ephemerals"; - case ZNODEEXISTS: return "Node exists"; - case ZNOTEMPTY: return "Not empty"; - case ZSESSIONEXPIRED: return "Session expired"; - case ZINVALIDCALLBACK: return "Invalid callback"; - case ZINVALIDACL: return "Invalid ACL"; - case ZAUTHFAILED: return "Authentication failed"; - case ZCLOSING: return "ZooKeeper is closing"; - case ZNOTHING: return "(not error) no server responses to process"; - case ZSESSIONMOVED: return "Session moved to another server, so operation is ignored"; + case Error::ZOK: return "Ok"; + case Error::ZSYSTEMERROR: return "System error"; + case Error::ZRUNTIMEINCONSISTENCY: return "Run time inconsistency"; + case Error::ZDATAINCONSISTENCY: return "Data inconsistency"; + case Error::ZCONNECTIONLOSS: return "Connection loss"; + case Error::ZMARSHALLINGERROR: return "Marshalling error"; + case Error::ZUNIMPLEMENTED: return "Unimplemented"; + case Error::ZOPERATIONTIMEOUT: return "Operation timeout"; + case Error::ZBADARGUMENTS: return "Bad arguments"; + case Error::ZINVALIDSTATE: return "Invalid zhandle state"; + case Error::ZAPIERROR: return "API error"; + case Error::ZNONODE: return "No node"; + case Error::ZNOAUTH: return "Not authenticated"; + case Error::ZBADVERSION: return "Bad version"; + case Error::ZNOCHILDRENFOREPHEMERALS: return "No children for ephemerals"; + case Error::ZNODEEXISTS: return "Node exists"; + case Error::ZNOTEMPTY: return "Not empty"; + case Error::ZSESSIONEXPIRED: return "Session expired"; + case Error::ZINVALIDCALLBACK: return "Invalid callback"; + case Error::ZINVALIDACL: return "Invalid ACL"; + case Error::ZAUTHFAILED: return "Authentication failed"; + case Error::ZCLOSING: return "ZooKeeper is closing"; + case Error::ZNOTHING: return "(not error) no server responses to process"; + case Error::ZSESSIONMOVED: return "Session moved to another server, so operation is ignored"; } - if (code > 0) - return strerror(code); - return "unknown error"; + __builtin_unreachable(); } -bool isHardwareError(int32_t zk_return_code) +bool isHardwareError(Error zk_return_code) { - return zk_return_code == ZINVALIDSTATE - || zk_return_code == ZSESSIONEXPIRED - || zk_return_code == ZSESSIONMOVED - || zk_return_code == ZCONNECTIONLOSS - || zk_return_code == ZMARSHALLINGERROR - || zk_return_code == ZOPERATIONTIMEOUT; + return zk_return_code == Error::ZINVALIDSTATE + || zk_return_code == Error::ZSESSIONEXPIRED + || zk_return_code == Error::ZSESSIONMOVED + || zk_return_code == Error::ZCONNECTIONLOSS + || zk_return_code == Error::ZMARSHALLINGERROR + || zk_return_code == Error::ZOPERATIONTIMEOUT; } -bool isUserError(int32_t zk_return_code) +bool isUserError(Error zk_return_code) { - return zk_return_code == ZNONODE - || zk_return_code == ZBADVERSION - || zk_return_code == ZNOCHILDRENFOREPHEMERALS - || zk_return_code == ZNODEEXISTS - || zk_return_code == ZNOTEMPTY; + return zk_return_code == Error::ZNONODE + || zk_return_code == Error::ZBADVERSION + || zk_return_code == Error::ZNOCHILDRENFOREPHEMERALS + || zk_return_code == Error::ZNODEEXISTS + || zk_return_code == Error::ZNOTEMPTY; } diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index f415e0306e8..409c3838147 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -53,6 +53,57 @@ struct Stat int64_t pzxid; }; +enum class Error : int32_t +{ + ZOK = 0, + + /** System and server-side errors. + * This is never thrown by the server, it shouldn't be used other than + * to indicate a range. Specifically error codes greater than this + * value, but lesser than ZAPIERROR, are system errors. + */ + ZSYSTEMERROR = -1, + + ZRUNTIMEINCONSISTENCY = -2, /// A runtime inconsistency was found + ZDATAINCONSISTENCY = -3, /// A data inconsistency was found + ZCONNECTIONLOSS = -4, /// Connection to the server has been lost + ZMARSHALLINGERROR = -5, /// Error while marshalling or unmarshalling data + ZUNIMPLEMENTED = -6, /// Operation is unimplemented + ZOPERATIONTIMEOUT = -7, /// Operation timeout + ZBADARGUMENTS = -8, /// Invalid arguments + ZINVALIDSTATE = -9, /// Invliad zhandle state + + /** API errors. + * This is never thrown by the server, it shouldn't be used other than + * to indicate a range. Specifically error codes greater than this + * value are API errors. + */ + ZAPIERROR = -100, + + ZNONODE = -101, /// Node does not exist + ZNOAUTH = -102, /// Not authenticated + ZBADVERSION = -103, /// Version conflict + ZNOCHILDRENFOREPHEMERALS = -108, /// Ephemeral nodes may not have children + ZNODEEXISTS = -110, /// The node already exists + ZNOTEMPTY = -111, /// The node has children + ZSESSIONEXPIRED = -112, /// The session has been expired by the server + ZINVALIDCALLBACK = -113, /// Invalid callback specified + ZINVALIDACL = -114, /// Invalid ACL specified + ZAUTHFAILED = -115, /// Client authentication failed + ZCLOSING = -116, /// ZooKeeper is closing + ZNOTHING = -117, /// (not error) no server responses to process + ZSESSIONMOVED = -118 /// Session moved to another server, so operation is ignored +}; + +/// Network errors and similar. You should reinitialize ZooKeeper session in case of these errors +bool isHardwareError(Error code); + +/// Valid errors sent from the server about database state (like "no node"). Logical and authentication errors (like "bad arguments") are not here. +bool isUserError(Error code); + +const char * errorMessage(Error code); + + struct Request; using RequestPtr = std::shared_ptr; using Requests = std::vector; @@ -74,7 +125,7 @@ using ResponseCallback = std::function; struct Response { - int32_t error = 0; + Error error = Error::ZOK; Response() = default; Response(const Response &) = default; Response & operator=(const Response &) = default; @@ -225,56 +276,6 @@ using CheckCallback = std::function; using MultiCallback = std::function; -enum Error -{ - ZOK = 0, - - /** System and server-side errors. - * This is never thrown by the server, it shouldn't be used other than - * to indicate a range. Specifically error codes greater than this - * value, but lesser than ZAPIERROR, are system errors. - */ - ZSYSTEMERROR = -1, - - ZRUNTIMEINCONSISTENCY = -2, /// A runtime inconsistency was found - ZDATAINCONSISTENCY = -3, /// A data inconsistency was found - ZCONNECTIONLOSS = -4, /// Connection to the server has been lost - ZMARSHALLINGERROR = -5, /// Error while marshalling or unmarshalling data - ZUNIMPLEMENTED = -6, /// Operation is unimplemented - ZOPERATIONTIMEOUT = -7, /// Operation timeout - ZBADARGUMENTS = -8, /// Invalid arguments - ZINVALIDSTATE = -9, /// Invliad zhandle state - - /** API errors. - * This is never thrown by the server, it shouldn't be used other than - * to indicate a range. Specifically error codes greater than this - * value are API errors. - */ - ZAPIERROR = -100, - - ZNONODE = -101, /// Node does not exist - ZNOAUTH = -102, /// Not authenticated - ZBADVERSION = -103, /// Version conflict - ZNOCHILDRENFOREPHEMERALS = -108, /// Ephemeral nodes may not have children - ZNODEEXISTS = -110, /// The node already exists - ZNOTEMPTY = -111, /// The node has children - ZSESSIONEXPIRED = -112, /// The session has been expired by the server - ZINVALIDCALLBACK = -113, /// Invalid callback specified - ZINVALIDACL = -114, /// Invalid ACL specified - ZAUTHFAILED = -115, /// Client authentication failed - ZCLOSING = -116, /// ZooKeeper is closing - ZNOTHING = -117, /// (not error) no server responses to process - ZSESSIONMOVED = -118 /// Session moved to another server, so operation is ignored -}; - -/// Network errors and similar. You should reinitialize ZooKeeper session in case of these errors -bool isHardwareError(int32_t code); - -/// Valid errors sent from the server about database state (like "no node"). Logical and authentication errors (like "bad arguments") are not here. -bool isUserError(int32_t code); - -const char * errorMessage(int32_t code); - /// For watches. enum State { @@ -301,19 +302,19 @@ class Exception : public DB::Exception { private: /// Delegate constructor, used to minimize repetition; last parameter used for overload resolution. - Exception(const std::string & msg, const int32_t code_, int); + Exception(const std::string & msg, const Error code_, int); public: - explicit Exception(const int32_t code_); - Exception(const std::string & msg, const int32_t code_); - Exception(const int32_t code_, const std::string & path); + explicit Exception(const Error code_); + Exception(const std::string & msg, const Error code_); + Exception(const Error code_, const std::string & path); Exception(const Exception & exc); const char * name() const throw() override { return "Coordination::Exception"; } const char * className() const throw() override { return "Coordination::Exception"; } Exception * clone() const override { return new Exception(*this); } - const int32_t code; + const Error code; }; diff --git a/src/Common/ZooKeeper/Increment.h b/src/Common/ZooKeeper/Increment.h index fa5f550ca9b..883fc00442e 100644 --- a/src/Common/ZooKeeper/Increment.h +++ b/src/Common/ZooKeeper/Increment.h @@ -29,11 +29,11 @@ public: if (zookeeper->tryGet(path, result_str, &stat)) { result = std::stol(result_str) + 1; - success = zookeeper->trySet(path, std::to_string(result), stat.version) == Coordination::ZOK; + success = zookeeper->trySet(path, std::to_string(result), stat.version) == Coordination::Error::ZOK; } else { - success = zookeeper->tryCreate(path, std::to_string(result), zkutil::CreateMode::Persistent) == Coordination::ZOK; + success = zookeeper->tryCreate(path, std::to_string(result), zkutil::CreateMode::Persistent) == Coordination::Error::ZOK; } } while (!success); diff --git a/src/Common/ZooKeeper/KeeperException.h b/src/Common/ZooKeeper/KeeperException.h index 5fcca4cf3d2..6498aca809c 100644 --- a/src/Common/ZooKeeper/KeeperException.h +++ b/src/Common/ZooKeeper/KeeperException.h @@ -21,12 +21,12 @@ public: /// If it is user error throws KeeperMultiException else throws ordinary KeeperException /// If it is ZOK does nothing - static void check(int32_t code, const Coordination::Requests & requests, const Coordination::Responses & responses); + static void check(Coordination::Error code, const Coordination::Requests & requests, const Coordination::Responses & responses); - KeeperMultiException(int32_t code, const Coordination::Requests & requests, const Coordination::Responses & responses); + KeeperMultiException(Coordination::Error code, const Coordination::Requests & requests, const Coordination::Responses & responses); private: - static size_t getFailedOpIndex(int32_t code, const Coordination::Responses & responses); + static size_t getFailedOpIndex(Coordination::Error code, const Coordination::Responses & responses); }; } diff --git a/src/Common/ZooKeeper/LeaderElection.h b/src/Common/ZooKeeper/LeaderElection.h index e3b97e7f8ca..dca87efe7c2 100644 --- a/src/Common/ZooKeeper/LeaderElection.h +++ b/src/Common/ZooKeeper/LeaderElection.h @@ -121,7 +121,7 @@ private: { DB::tryLogCurrentException(log); - if (e.code == Coordination::ZSESSIONEXPIRED) + if (e.code == Coordination::Error::ZSESSIONEXPIRED) return; } catch (...) diff --git a/src/Common/ZooKeeper/Lock.cpp b/src/Common/ZooKeeper/Lock.cpp index c781d8ba2bf..9c966cc576d 100644 --- a/src/Common/ZooKeeper/Lock.cpp +++ b/src/Common/ZooKeeper/Lock.cpp @@ -16,13 +16,13 @@ bool Lock::tryLock() else { std::string dummy; - int32_t code = zookeeper->tryCreate(lock_path, lock_message, zkutil::CreateMode::Ephemeral, dummy); + Coordination::Error code = zookeeper->tryCreate(lock_path, lock_message, zkutil::CreateMode::Ephemeral, dummy); - if (code == Coordination::ZNODEEXISTS) + if (code == Coordination::Error::ZNODEEXISTS) { locked.reset(); } - else if (code == Coordination::ZOK) + else if (code == Coordination::Error::ZOK) { locked = std::make_unique(zookeeper); } diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 4f736e66aab..a734d218ff6 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -158,7 +158,7 @@ struct TestKeeperMultiRequest final : MultiRequest, TestKeeperRequest requests.push_back(std::make_shared(*concrete_request_check)); } else - throw Exception("Illegal command as part of multi ZooKeeper request", ZBADARGUMENTS); + throw Exception("Illegal command as part of multi ZooKeeper request", Error::ZBADARGUMENTS); } } @@ -338,7 +338,7 @@ ResponsePtr TestKeeperListRequest::process(TestKeeper::Container & container, in { auto path_prefix = path; if (path_prefix.empty()) - throw Exception("Logical error: path cannot be empty", ZSESSIONEXPIRED); + throw Exception("Logical error: path cannot be empty", Error::ZSESSIONEXPIRED); if (path_prefix.back() != '/') path_prefix += '/'; @@ -514,7 +514,7 @@ void TestKeeper::finalize() WatchResponse response; response.type = SESSION; response.state = EXPIRED_SESSION; - response.error = ZSESSIONEXPIRED; + response.error = Error::ZSESSIONEXPIRED; for (auto & callback : path_watch.second) { @@ -541,7 +541,7 @@ void TestKeeper::finalize() if (info.callback) { ResponsePtr response = info.request->createResponse(); - response->error = ZSESSIONEXPIRED; + response->error = Error::ZSESSIONEXPIRED; try { info.callback(*response); @@ -556,7 +556,7 @@ void TestKeeper::finalize() WatchResponse response; response.type = SESSION; response.state = EXPIRED_SESSION; - response.error = ZSESSIONEXPIRED; + response.error = Error::ZSESSIONEXPIRED; try { info.watch(response); @@ -587,10 +587,10 @@ void TestKeeper::pushRequest(RequestInfo && request) std::lock_guard lock(push_request_mutex); if (expired) - throw Exception("Session expired", ZSESSIONEXPIRED); + throw Exception("Session expired", Error::ZSESSIONEXPIRED); if (!requests_queue.tryPush(std::move(request), operation_timeout.totalMilliseconds())) - throw Exception("Cannot push request to queue within operation timeout", ZOPERATIONTIMEOUT); + throw Exception("Cannot push request to queue within operation timeout", Error::ZOPERATIONTIMEOUT); } catch (...) { diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 115518e2bf9..169299483ce 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -38,9 +38,9 @@ const int CreateMode::PersistentSequential = 2; const int CreateMode::EphemeralSequential = 3; -static void check(int32_t code, const std::string & path) +static void check(Coordination::Error code, const std::string & path) { - if (code) + if (code != Coordination::Error::ZOK) throw KeeperException(code, path); } @@ -59,7 +59,7 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho if (implementation == "zookeeper") { if (hosts.empty()) - throw KeeperException("No hosts passed to ZooKeeper constructor.", Coordination::ZBADARGUMENTS); + throw KeeperException("No hosts passed to ZooKeeper constructor.", Coordination::Error::ZBADARGUMENTS); std::vector hosts_strings; splitInto<','>(hosts_strings, hosts); @@ -84,7 +84,7 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho } if (nodes.empty()) - throw KeeperException("Cannot use any of provided ZooKeeper nodes", Coordination::ZBADARGUMENTS); + throw KeeperException("Cannot use any of provided ZooKeeper nodes", Coordination::Error::ZBADARGUMENTS); impl = std::make_unique( nodes, @@ -112,7 +112,7 @@ void ZooKeeper::init(const std::string & implementation_, const std::string & ho } if (!chroot.empty() && !exists("/")) - throw KeeperException("Zookeeper root doesn't exist. You should create root node " + chroot + " before start.", Coordination::ZNONODE); + throw KeeperException("Zookeeper root doesn't exist. You should create root node " + chroot + " before start.", Coordination::Error::ZNONODE); } ZooKeeper::ZooKeeper(const std::string & hosts_, const std::string & identity_, int32_t session_timeout_ms_, @@ -164,7 +164,7 @@ struct ZooKeeperArgs implementation = config.getString(config_name + "." + key); } else - throw KeeperException(std::string("Unknown key ") + key + " in config file", Coordination::ZBADARGUMENTS); + throw KeeperException(std::string("Unknown key ") + key + " in config file", Coordination::Error::ZBADARGUMENTS); } /// Shuffle the hosts to distribute the load among ZooKeeper nodes. @@ -182,7 +182,7 @@ struct ZooKeeperArgs if (!chroot.empty()) { if (chroot.front() != '/') - throw KeeperException(std::string("Root path in config file should start with '/', but got ") + chroot, Coordination::ZBADARGUMENTS); + throw KeeperException(std::string("Root path in config file should start with '/', but got ") + chroot, Coordination::Error::ZBADARGUMENTS); if (chroot.back() == '/') chroot.pop_back(); } @@ -211,17 +211,17 @@ static Coordination::WatchCallback callbackForEvent(const EventPtr & watch) } -int32_t ZooKeeper::getChildrenImpl(const std::string & path, Strings & res, +Coordination::Error ZooKeeper::getChildrenImpl(const std::string & path, Strings & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) { - int32_t code = 0; + Coordination::Error code = Coordination::Error::ZOK; Poco::Event event; auto callback = [&](const Coordination::ListResponse & response) { code = response.error; - if (!code) + if (code == Coordination::Error::ZOK) { res = response.names; if (stat) @@ -251,37 +251,37 @@ Strings ZooKeeper::getChildrenWatch( return res; } -int32_t ZooKeeper::tryGetChildren(const std::string & path, Strings & res, +Coordination::Error ZooKeeper::tryGetChildren(const std::string & path, Strings & res, Coordination::Stat * stat, const EventPtr & watch) { - int32_t code = getChildrenImpl(path, res, stat, callbackForEvent(watch)); + Coordination::Error code = getChildrenImpl(path, res, stat, callbackForEvent(watch)); - if (!(code == Coordination::ZOK || code == Coordination::ZNONODE)) + if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE)) throw KeeperException(code, path); return code; } -int32_t ZooKeeper::tryGetChildrenWatch(const std::string & path, Strings & res, +Coordination::Error ZooKeeper::tryGetChildrenWatch(const std::string & path, Strings & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) { - int32_t code = getChildrenImpl(path, res, stat, watch_callback); + Coordination::Error code = getChildrenImpl(path, res, stat, watch_callback); - if (!(code == Coordination::ZOK || code == Coordination::ZNONODE)) + if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE)) throw KeeperException(code, path); return code; } -int32_t ZooKeeper::createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created) +Coordination::Error ZooKeeper::createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created) { - int32_t code = 0; + Coordination::Error code = Coordination::Error::ZOK; Poco::Event event; auto callback = [&](const Coordination::CreateResponse & response) { code = response.error; - if (!code) + if (code == Coordination::Error::ZOK) path_created = response.path_created; event.set(); }; @@ -298,20 +298,20 @@ std::string ZooKeeper::create(const std::string & path, const std::string & data return path_created; } -int32_t ZooKeeper::tryCreate(const std::string & path, const std::string & data, int32_t mode, std::string & path_created) +Coordination::Error ZooKeeper::tryCreate(const std::string & path, const std::string & data, int32_t mode, std::string & path_created) { - int32_t code = createImpl(path, data, mode, path_created); + Coordination::Error code = createImpl(path, data, mode, path_created); - if (!(code == Coordination::ZOK || - code == Coordination::ZNONODE || - code == Coordination::ZNODEEXISTS || - code == Coordination::ZNOCHILDRENFOREPHEMERALS)) + if (!(code == Coordination::Error::ZOK || + code == Coordination::Error::ZNONODE || + code == Coordination::Error::ZNODEEXISTS || + code == Coordination::Error::ZNOCHILDRENFOREPHEMERALS)) throw KeeperException(code, path); return code; } -int32_t ZooKeeper::tryCreate(const std::string & path, const std::string & data, int32_t mode) +Coordination::Error ZooKeeper::tryCreate(const std::string & path, const std::string & data, int32_t mode) { std::string path_created; return tryCreate(path, data, mode, path_created); @@ -320,9 +320,9 @@ int32_t ZooKeeper::tryCreate(const std::string & path, const std::string & data, void ZooKeeper::createIfNotExists(const std::string & path, const std::string & data) { std::string path_created; - int32_t code = createImpl(path, data, CreateMode::Persistent, path_created); + Coordination::Error code = createImpl(path, data, CreateMode::Persistent, path_created); - if (code == Coordination::ZOK || code == Coordination::ZNODEEXISTS) + if (code == Coordination::Error::ZOK || code == Coordination::Error::ZNODEEXISTS) return; else throw KeeperException(code, path); @@ -341,14 +341,14 @@ void ZooKeeper::createAncestors(const std::string & path) } } -int32_t ZooKeeper::removeImpl(const std::string & path, int32_t version) +Coordination::Error ZooKeeper::removeImpl(const std::string & path, int32_t version) { - int32_t code = 0; + Coordination::Error code = Coordination::Error::ZOK; Poco::Event event; auto callback = [&](const Coordination::RemoveResponse & response) { - if (response.error) + if (response.error != Coordination::Error::ZOK) code = response.error; event.set(); }; @@ -363,26 +363,26 @@ void ZooKeeper::remove(const std::string & path, int32_t version) check(tryRemove(path, version), path); } -int32_t ZooKeeper::tryRemove(const std::string & path, int32_t version) +Coordination::Error ZooKeeper::tryRemove(const std::string & path, int32_t version) { - int32_t code = removeImpl(path, version); - if (!(code == Coordination::ZOK || - code == Coordination::ZNONODE || - code == Coordination::ZBADVERSION || - code == Coordination::ZNOTEMPTY)) + Coordination::Error code = removeImpl(path, version); + if (!(code == Coordination::Error::ZOK || + code == Coordination::Error::ZNONODE || + code == Coordination::Error::ZBADVERSION || + code == Coordination::Error::ZNOTEMPTY)) throw KeeperException(code, path); return code; } -int32_t ZooKeeper::existsImpl(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) +Coordination::Error ZooKeeper::existsImpl(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) { - int32_t code = 0; + Coordination::Error code = Coordination::Error::ZOK; Poco::Event event; auto callback = [&](const Coordination::ExistsResponse & response) { code = response.error; - if (!code && stat) + if (code == Coordination::Error::ZOK && stat) *stat = response.stat; event.set(); }; @@ -399,22 +399,22 @@ bool ZooKeeper::exists(const std::string & path, Coordination::Stat * stat, cons bool ZooKeeper::existsWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) { - int32_t code = existsImpl(path, stat, watch_callback); + Coordination::Error code = existsImpl(path, stat, watch_callback); - if (!(code == Coordination::ZOK || code == Coordination::ZNONODE)) + if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE)) throw KeeperException(code, path); - return code != Coordination::ZNONODE; + return code != Coordination::Error::ZNONODE; } -int32_t ZooKeeper::getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) +Coordination::Error ZooKeeper::getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) { - int32_t code = 0; + Coordination::Error code = Coordination::Error::ZOK; Poco::Event event; auto callback = [&](const Coordination::GetResponse & response) { code = response.error; - if (!code) + if (code == Coordination::Error::ZOK) { res = response.data; if (stat) @@ -431,7 +431,7 @@ int32_t ZooKeeper::getImpl(const std::string & path, std::string & res, Coordina std::string ZooKeeper::get(const std::string & path, Coordination::Stat * stat, const EventPtr & watch) { - int32_t code = 0; + Coordination::Error code = Coordination::Error::ZOK; std::string res; if (tryGet(path, res, stat, watch, &code)) return res; @@ -441,7 +441,7 @@ std::string ZooKeeper::get(const std::string & path, Coordination::Stat * stat, std::string ZooKeeper::getWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) { - int32_t code = 0; + Coordination::Error code = Coordination::Error::ZOK; std::string res; if (tryGetWatch(path, res, stat, watch_callback, &code)) return res; @@ -449,34 +449,44 @@ std::string ZooKeeper::getWatch(const std::string & path, Coordination::Stat * s throw KeeperException("Can't get data for node " + path + ": node doesn't exist", code); } -bool ZooKeeper::tryGet(const std::string & path, std::string & res, Coordination::Stat * stat, const EventPtr & watch, int * return_code) +bool ZooKeeper::tryGet( + const std::string & path, + std::string & res, + Coordination::Stat * stat, + const EventPtr & watch, + Coordination::Error * return_code) { return tryGetWatch(path, res, stat, callbackForEvent(watch), return_code); } -bool ZooKeeper::tryGetWatch(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback, int * return_code) +bool ZooKeeper::tryGetWatch( + const std::string & path, + std::string & res, + Coordination::Stat * stat, + Coordination::WatchCallback watch_callback, + Coordination::Error * return_code) { - int32_t code = getImpl(path, res, stat, watch_callback); + Coordination::Error code = getImpl(path, res, stat, watch_callback); - if (!(code == Coordination::ZOK || code == Coordination::ZNONODE)) + if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE)) throw KeeperException(code, path); if (return_code) *return_code = code; - return code == Coordination::ZOK; + return code == Coordination::Error::ZOK; } -int32_t ZooKeeper::setImpl(const std::string & path, const std::string & data, +Coordination::Error ZooKeeper::setImpl(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat) { - int32_t code = 0; + Coordination::Error code = Coordination::Error::ZOK; Poco::Event event; auto callback = [&](const Coordination::SetResponse & response) { code = response.error; - if (!code && stat) + if (code == Coordination::Error::ZOK && stat) *stat = response.stat; event.set(); }; @@ -493,34 +503,34 @@ void ZooKeeper::set(const std::string & path, const std::string & data, int32_t void ZooKeeper::createOrUpdate(const std::string & path, const std::string & data, int32_t mode) { - int32_t code = trySet(path, data, -1); - if (code == Coordination::ZNONODE) + Coordination::Error code = trySet(path, data, -1); + if (code == Coordination::Error::ZNONODE) { create(path, data, mode); } - else if (code != Coordination::ZOK) + else if (code != Coordination::Error::ZOK) throw KeeperException(code, path); } -int32_t ZooKeeper::trySet(const std::string & path, const std::string & data, +Coordination::Error ZooKeeper::trySet(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat) { - int32_t code = setImpl(path, data, version, stat); + Coordination::Error code = setImpl(path, data, version, stat); - if (!(code == Coordination::ZOK || - code == Coordination::ZNONODE || - code == Coordination::ZBADVERSION)) + if (!(code == Coordination::Error::ZOK || + code == Coordination::Error::ZNONODE || + code == Coordination::Error::ZBADVERSION)) throw KeeperException(code, path); return code; } -int32_t ZooKeeper::multiImpl(const Coordination::Requests & requests, Coordination::Responses & responses) +Coordination::Error ZooKeeper::multiImpl(const Coordination::Requests & requests, Coordination::Responses & responses) { if (requests.empty()) - return Coordination::ZOK; + return Coordination::Error::ZOK; - int32_t code = 0; + Coordination::Error code = Coordination::Error::ZOK; Poco::Event event; auto callback = [&](const Coordination::MultiResponse & response) @@ -538,15 +548,15 @@ int32_t ZooKeeper::multiImpl(const Coordination::Requests & requests, Coordinati Coordination::Responses ZooKeeper::multi(const Coordination::Requests & requests) { Coordination::Responses responses; - int32_t code = multiImpl(requests, responses); + Coordination::Error code = multiImpl(requests, responses); KeeperMultiException::check(code, requests, responses); return responses; } -int32_t ZooKeeper::tryMulti(const Coordination::Requests & requests, Coordination::Responses & responses) +Coordination::Error ZooKeeper::tryMulti(const Coordination::Requests & requests, Coordination::Responses & responses) { - int32_t code = multiImpl(requests, responses); - if (code && !Coordination::isUserError(code)) + Coordination::Error code = multiImpl(requests, responses); + if (code != Coordination::Error::ZOK && !Coordination::isUserError(code)) throw KeeperException(code); return code; } @@ -587,7 +597,7 @@ void ZooKeeper::removeChildrenRecursive(const std::string & path) void ZooKeeper::tryRemoveChildrenRecursive(const std::string & path) { Strings children; - if (tryGetChildren(path, children) != Coordination::ZOK) + if (tryGetChildren(path, children) != Coordination::Error::ZOK) return; while (!children.empty()) { @@ -609,7 +619,7 @@ void ZooKeeper::tryRemoveChildrenRecursive(const std::string & path) /// this means someone is concurrently removing these children and we will have /// to remove them one by one. Coordination::Responses responses; - if (tryMulti(ops, responses) != Coordination::ZOK) + if (tryMulti(ops, responses) != Coordination::Error::ZOK) for (const std::string & child : batch) tryRemove(child); } @@ -645,7 +655,7 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition & auto callback = [state](const Coordination::ExistsResponse & response) { - state->code = response.error; + state->code = int32_t(response.error); if (state->code) state->event.set(); }; @@ -654,7 +664,7 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition & { if (!state->code) { - state->code = response.error; + state->code = int32_t(response.error); if (!state->code) state->event_type = response.type; state->event.set(); @@ -670,11 +680,11 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition & else if (!state->event.tryWait(1000)) continue; - if (state->code == Coordination::ZNONODE) + if (state->code == int32_t(Coordination::Error::ZNONODE)) return true; if (state->code) - throw KeeperException(state->code, path); + throw KeeperException(static_cast(state->code.load(std::memory_order_seq_cst)), path); if (state->event_type == Coordination::DELETED) return true; @@ -688,11 +698,6 @@ ZooKeeperPtr ZooKeeper::startNewSession() const } -std::string ZooKeeper::error2string(int32_t code) -{ - return Coordination::errorMessage(code); -} - bool ZooKeeper::expired() { return impl->isExpired(); @@ -712,7 +717,7 @@ std::future ZooKeeper::asyncCreate(const std::stri auto callback = [promise, path](const Coordination::CreateResponse & response) mutable { - if (response.error) + if (response.error != Coordination::Error::ZOK) promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error))); else promise->set_value(response); @@ -730,7 +735,7 @@ std::future ZooKeeper::asyncGet(const std::string & p auto callback = [promise, path](const Coordination::GetResponse & response) mutable { - if (response.error) + if (response.error != Coordination::Error::ZOK) promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error))); else promise->set_value(response); @@ -748,7 +753,7 @@ std::future ZooKeeper::asyncTryGet(const std::string auto callback = [promise, path](const Coordination::GetResponse & response) mutable { - if (response.error && response.error != Coordination::ZNONODE) + if (response.error != Coordination::Error::ZOK && response.error != Coordination::Error::ZNONODE) promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error))); else promise->set_value(response); @@ -765,7 +770,7 @@ std::future ZooKeeper::asyncExists(const std::stri auto callback = [promise, path](const Coordination::ExistsResponse & response) mutable { - if (response.error && response.error != Coordination::ZNONODE) + if (response.error != Coordination::Error::ZOK && response.error != Coordination::Error::ZNONODE) promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error))); else promise->set_value(response); @@ -782,7 +787,7 @@ std::future ZooKeeper::asyncSet(const std::string & p auto callback = [promise, path](const Coordination::SetResponse & response) mutable { - if (response.error) + if (response.error != Coordination::Error::ZOK) promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error))); else promise->set_value(response); @@ -799,7 +804,7 @@ std::future ZooKeeper::asyncGetChildren(const std::s auto callback = [promise, path](const Coordination::ListResponse & response) mutable { - if (response.error) + if (response.error != Coordination::Error::ZOK) promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error))); else promise->set_value(response); @@ -816,7 +821,7 @@ std::future ZooKeeper::asyncRemove(const std::stri auto callback = [promise, path](const Coordination::RemoveResponse & response) mutable { - if (response.error) + if (response.error != Coordination::Error::ZOK) promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error))); else promise->set_value(response); @@ -833,8 +838,13 @@ std::future ZooKeeper::asyncTryRemove(const std::s auto callback = [promise, path](const Coordination::RemoveResponse & response) mutable { - if (response.error && response.error != Coordination::ZNONODE && response.error != Coordination::ZBADVERSION && response.error != Coordination::ZNOTEMPTY) + if (response.error != Coordination::Error::ZOK + && response.error != Coordination::Error::ZNONODE + && response.error != Coordination::Error::ZBADVERSION + && response.error != Coordination::Error::ZNOTEMPTY) + { promise->set_exception(std::make_exception_ptr(KeeperException(path, response.error))); + } else promise->set_value(response); }; @@ -864,7 +874,7 @@ std::future ZooKeeper::asyncMulti(const Coordinatio auto callback = [promise](const Coordination::MultiResponse & response) mutable { - if (response.error) + if (response.error != Coordination::Error::ZOK) promise->set_exception(std::make_exception_ptr(KeeperException(response.error))); else promise->set_value(response); @@ -874,7 +884,7 @@ std::future ZooKeeper::asyncMulti(const Coordinatio return future; } -int32_t ZooKeeper::tryMultiNoThrow(const Coordination::Requests & requests, Coordination::Responses & responses) +Coordination::Error ZooKeeper::tryMultiNoThrow(const Coordination::Requests & requests, Coordination::Responses & responses) { try { @@ -887,24 +897,24 @@ int32_t ZooKeeper::tryMultiNoThrow(const Coordination::Requests & requests, Coor } -size_t KeeperMultiException::getFailedOpIndex(int32_t exception_code, const Coordination::Responses & responses) +size_t KeeperMultiException::getFailedOpIndex(Coordination::Error exception_code, const Coordination::Responses & responses) { if (responses.empty()) throw DB::Exception("Responses for multi transaction is empty", DB::ErrorCodes::LOGICAL_ERROR); for (size_t index = 0, size = responses.size(); index < size; ++index) - if (responses[index]->error) + if (responses[index]->error != Coordination::Error::ZOK) return index; if (!Coordination::isUserError(exception_code)) - throw DB::Exception("There are no failed OPs because '" + ZooKeeper::error2string(exception_code) + "' is not valid response code for that", + throw DB::Exception("There are no failed OPs because '" + std::string(Coordination::errorMessage(exception_code)) + "' is not valid response code for that", DB::ErrorCodes::LOGICAL_ERROR); throw DB::Exception("There is no failed OpResult", DB::ErrorCodes::LOGICAL_ERROR); } -KeeperMultiException::KeeperMultiException(int32_t exception_code, const Coordination::Requests & requests_, const Coordination::Responses & responses_) +KeeperMultiException::KeeperMultiException(Coordination::Error exception_code, const Coordination::Requests & requests_, const Coordination::Responses & responses_) : KeeperException("Transaction failed", exception_code), requests(requests_), responses(responses_), failed_op_index(getFailedOpIndex(exception_code, responses)) { @@ -917,9 +927,10 @@ std::string KeeperMultiException::getPathForFirstFailedOp() const return requests[failed_op_index]->getPath(); } -void KeeperMultiException::check(int32_t exception_code, const Coordination::Requests & requests, const Coordination::Responses & responses) +void KeeperMultiException::check( + Coordination::Error exception_code, const Coordination::Requests & requests, const Coordination::Responses & responses) { - if (!exception_code) + if (exception_code == Coordination::Error::ZOK) return; if (Coordination::isUserError(exception_code)) diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 3bf9ad3c100..416e40c2da4 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -99,8 +99,8 @@ public: /// * The parent is ephemeral. /// * The node already exists. /// In case of other errors throws an exception. - int32_t tryCreate(const std::string & path, const std::string & data, int32_t mode, std::string & path_created); - int32_t tryCreate(const std::string & path, const std::string & data, int32_t mode); + Coordination::Error tryCreate(const std::string & path, const std::string & data, int32_t mode, std::string & path_created); + Coordination::Error tryCreate(const std::string & path, const std::string & data, int32_t mode); /// Create a Persistent node. /// Does nothing if the node already exists. @@ -117,7 +117,7 @@ public: /// * The node doesn't exist /// * Versions don't match /// * The node has children. - int32_t tryRemove(const std::string & path, int32_t version = -1); + Coordination::Error tryRemove(const std::string & path, int32_t version = -1); bool exists(const std::string & path, Coordination::Stat * stat = nullptr, const EventPtr & watch = nullptr); bool existsWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback); @@ -127,9 +127,11 @@ public: /// Doesn't not throw in the following cases: /// * The node doesn't exist. Returns false in this case. - bool tryGet(const std::string & path, std::string & res, Coordination::Stat * stat = nullptr, const EventPtr & watch = nullptr, int * code = nullptr); + bool tryGet(const std::string & path, std::string & res, Coordination::Stat * stat = nullptr, const EventPtr & watch = nullptr, + Coordination::Error * code = nullptr); - bool tryGetWatch(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback, int * code = nullptr); + bool tryGetWatch(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback, + Coordination::Error * code = nullptr); void set(const std::string & path, const std::string & data, int32_t version = -1, Coordination::Stat * stat = nullptr); @@ -140,7 +142,7 @@ public: /// Doesn't not throw in the following cases: /// * The node doesn't exist. /// * Versions do not match. - int32_t trySet(const std::string & path, const std::string & data, + Coordination::Error trySet(const std::string & path, const std::string & data, int32_t version = -1, Coordination::Stat * stat = nullptr); Strings getChildren(const std::string & path, @@ -153,11 +155,11 @@ public: /// Doesn't not throw in the following cases: /// * The node doesn't exist. - int32_t tryGetChildren(const std::string & path, Strings & res, + Coordination::Error tryGetChildren(const std::string & path, Strings & res, Coordination::Stat * stat = nullptr, const EventPtr & watch = nullptr); - int32_t tryGetChildrenWatch(const std::string & path, Strings & res, + Coordination::Error tryGetChildrenWatch(const std::string & path, Strings & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback); @@ -166,9 +168,9 @@ public: Coordination::Responses multi(const Coordination::Requests & requests); /// Throws only if some operation has returned an "unexpected" error /// - an error that would cause the corresponding try- method to throw. - int32_t tryMulti(const Coordination::Requests & requests, Coordination::Responses & responses); + Coordination::Error tryMulti(const Coordination::Requests & requests, Coordination::Responses & responses); /// Throws nothing (even session expired errors) - int32_t tryMultiNoThrow(const Coordination::Requests & requests, Coordination::Responses & responses); + Coordination::Error tryMultiNoThrow(const Coordination::Requests & requests, Coordination::Responses & responses); Int64 getClientID(); @@ -238,8 +240,6 @@ public: /// Like the previous one but don't throw any exceptions on future.get() FutureMulti tryAsyncMulti(const Coordination::Requests & ops); - static std::string error2string(int32_t code); - private: friend class EphemeralNodeHolder; @@ -250,13 +250,15 @@ private: void tryRemoveChildrenRecursive(const std::string & path); /// The following methods don't throw exceptions but return error codes. - int32_t createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created); - int32_t removeImpl(const std::string & path, int32_t version); - int32_t getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback); - int32_t setImpl(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat); - int32_t getChildrenImpl(const std::string & path, Strings & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback); - int32_t multiImpl(const Coordination::Requests & requests, Coordination::Responses & responses); - int32_t existsImpl(const std::string & path, Coordination::Stat * stat_, Coordination::WatchCallback watch_callback); + Coordination::Error createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created); + Coordination::Error removeImpl(const std::string & path, int32_t version); + Coordination::Error getImpl( + const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback); + Coordination::Error setImpl(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat); + Coordination::Error getChildrenImpl( + const std::string & path, Strings & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback); + Coordination::Error multiImpl(const Coordination::Requests & requests, Coordination::Responses & responses); + Coordination::Error existsImpl(const std::string & path, Coordination::Stat * stat_, Coordination::WatchCallback watch_callback); std::unique_ptr impl; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index e6cab23d2ce..8564b996bda 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -335,6 +335,13 @@ static void read(int32_t & x, ReadBuffer & in) x = __builtin_bswap32(x); } +static void read(Error & x, ReadBuffer & in) +{ + int32_t code; + readBinary(code, in); + x = Error(code); +} + static void read(bool & x, ReadBuffer & in) { readBinary(x, in); @@ -353,10 +360,10 @@ static void read(String & s, ReadBuffer & in) } if (size < 0) - throw Exception("Negative size while reading string from ZooKeeper", ZMARSHALLINGERROR); + throw Exception("Negative size while reading string from ZooKeeper", Error::ZMARSHALLINGERROR); if (size > MAX_STRING_OR_ARRAY_SIZE) - throw Exception("Too large string size while reading from ZooKeeper", ZMARSHALLINGERROR); + throw Exception("Too large string size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); s.resize(size); in.read(s.data(), size); @@ -367,7 +374,7 @@ template void read(std::array & s, ReadBuffer & in) int32_t size = 0; read(size, in); if (size != N) - throw Exception("Unexpected array size while reading from ZooKeeper", ZMARSHALLINGERROR); + throw Exception("Unexpected array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); in.read(s.data(), N); } @@ -391,9 +398,9 @@ template void read(std::vector & arr, ReadBuffer & in) int32_t size = 0; read(size, in); if (size < 0) - throw Exception("Negative size while reading array from ZooKeeper", ZMARSHALLINGERROR); + throw Exception("Negative size while reading array from ZooKeeper", Error::ZMARSHALLINGERROR); if (size > MAX_STRING_OR_ARRAY_SIZE) - throw Exception("Too large array size while reading from ZooKeeper", ZMARSHALLINGERROR); + throw Exception("Too large array size while reading from ZooKeeper", Error::ZMARSHALLINGERROR); arr.resize(size); for (auto & elem : arr) read(elem, in); @@ -489,7 +496,7 @@ struct ZooKeeperCloseResponse final : ZooKeeperResponse { void readImpl(ReadBuffer &) override { - throw Exception("Received response for close request", ZRUNTIMEINCONSISTENCY); + throw Exception("Received response for close request", Error::ZRUNTIMEINCONSISTENCY); } }; @@ -650,12 +657,12 @@ struct ZooKeeperErrorResponse final : ErrorResponse, ZooKeeperResponse { void readImpl(ReadBuffer & in) override { - int32_t read_error; + Coordination::Error read_error; Coordination::read(read_error, in); if (read_error != error) - throw Exception("Error code in ErrorResponse (" + toString(read_error) + ") doesn't match error code in header (" + toString(error) + ")", - ZMARSHALLINGERROR); + throw Exception(fmt::format("Error code in ErrorResponse ({}) doesn't match error code in header ({})", read_error, error), + Error::ZMARSHALLINGERROR); } }; @@ -691,7 +698,7 @@ struct ZooKeeperMultiRequest final : MultiRequest, ZooKeeperRequest requests.push_back(std::make_shared(*concrete_request_check)); } else - throw Exception("Illegal command as part of multi ZooKeeper request", ZBADARGUMENTS); + throw Exception("Illegal command as part of multi ZooKeeper request", Error::ZBADARGUMENTS); } } @@ -739,14 +746,14 @@ struct ZooKeeperMultiResponse final : MultiResponse, ZooKeeperResponse { ZooKeeper::OpNum op_num; bool done; - int32_t op_error; + Error op_error; Coordination::read(op_num, in); Coordination::read(done, in); Coordination::read(op_error, in); if (done) - throw Exception("Not enough results received for multi transaction", ZMARSHALLINGERROR); + throw Exception("Not enough results received for multi transaction", Error::ZMARSHALLINGERROR); /// op_num == -1 is special for multi transaction. /// For unknown reason, error code is duplicated in header and in response body. @@ -754,18 +761,18 @@ struct ZooKeeperMultiResponse final : MultiResponse, ZooKeeperResponse if (op_num == -1) response = std::make_shared(); - if (op_error) + if (op_error != Error::ZOK) { response->error = op_error; /// Set error for whole transaction. /// If some operations fail, ZK send global error as zero and then send details about each operation. /// It will set error code for first failed operation and it will set special "runtime inconsistency" code for other operations. - if (!error && op_error != ZRUNTIMEINCONSISTENCY) + if (error == Error::ZOK && op_error != Error::ZRUNTIMEINCONSISTENCY) error = op_error; } - if (!op_error || op_num == -1) + if (op_error == Error::ZOK || op_num == -1) dynamic_cast(*response).readImpl(in); } @@ -780,11 +787,11 @@ struct ZooKeeperMultiResponse final : MultiResponse, ZooKeeperResponse Coordination::read(error_read, in); if (!done) - throw Exception("Too many results received for multi transaction", ZMARSHALLINGERROR); + throw Exception("Too many results received for multi transaction", Error::ZMARSHALLINGERROR); if (op_num != -1) - throw Exception("Unexpected op_num received at the end of results for multi transaction", ZMARSHALLINGERROR); + throw Exception("Unexpected op_num received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); if (error_read != -1) - throw Exception("Unexpected error value received at the end of results for multi transaction", ZMARSHALLINGERROR); + throw Exception("Unexpected error value received at the end of results for multi transaction", Error::ZMARSHALLINGERROR); } } }; @@ -883,7 +890,7 @@ void ZooKeeper::connect( Poco::Timespan connection_timeout) { if (nodes.empty()) - throw Exception("No nodes passed to ZooKeeper constructor", ZBADARGUMENTS); + throw Exception("No nodes passed to ZooKeeper constructor", Error::ZBADARGUMENTS); static constexpr size_t num_tries = 3; bool connected = false; @@ -970,7 +977,7 @@ void ZooKeeper::connect( } message << fail_reasons.str() << "\n"; - throw Exception(message.str(), ZCONNECTIONLOSS); + throw Exception(message.str(), Error::ZCONNECTIONLOSS); } } @@ -1005,11 +1012,11 @@ void ZooKeeper::receiveHandshake() read(handshake_length); if (handshake_length != 36) - throw Exception("Unexpected handshake length received: " + toString(handshake_length), ZMARSHALLINGERROR); + throw Exception("Unexpected handshake length received: " + toString(handshake_length), Error::ZMARSHALLINGERROR); read(protocol_version_read); if (protocol_version_read != protocol_version) - throw Exception("Unexpected protocol version: " + toString(protocol_version_read), ZMARSHALLINGERROR); + throw Exception("Unexpected protocol version: " + toString(protocol_version_read), Error::ZMARSHALLINGERROR); read(timeout); if (timeout != session_timeout.totalMilliseconds()) @@ -1032,7 +1039,7 @@ void ZooKeeper::sendAuth(const String & scheme, const String & data) int32_t length; XID read_xid; int64_t zxid; - int32_t err; + Error err; read(length); size_t count_before_event = in->count(); @@ -1042,16 +1049,16 @@ void ZooKeeper::sendAuth(const String & scheme, const String & data) if (read_xid != auth_xid) throw Exception("Unexpected event received in reply to auth request: " + toString(read_xid), - ZMARSHALLINGERROR); + Error::ZMARSHALLINGERROR); int32_t actual_length = in->count() - count_before_event; if (length != actual_length) throw Exception("Response length doesn't match. Expected: " + toString(length) + ", actual: " + toString(actual_length), - ZMARSHALLINGERROR); + Error::ZMARSHALLINGERROR); - if (err) - throw Exception("Error received in reply to auth request. Code: " + toString(err) + ". Message: " + String(errorMessage(err)), - ZMARSHALLINGERROR); + if (err != Error::ZOK) + throw Exception("Error received in reply to auth request. Code: " + toString(int32_t(err)) + ". Message: " + String(errorMessage(err)), + Error::ZMARSHALLINGERROR); } @@ -1154,7 +1161,7 @@ void ZooKeeper::receiveThread() earliest_operation = operations.begin()->second; auto earliest_operation_deadline = earliest_operation->time + std::chrono::microseconds(operation_timeout.totalMicroseconds()); if (now > earliest_operation_deadline) - throw Exception("Operation timeout (deadline already expired) for path: " + earliest_operation->request->getPath(), ZOPERATIONTIMEOUT); + throw Exception("Operation timeout (deadline already expired) for path: " + earliest_operation->request->getPath(), Error::ZOPERATIONTIMEOUT); max_wait = std::chrono::duration_cast(earliest_operation_deadline - now).count(); } } @@ -1170,10 +1177,10 @@ void ZooKeeper::receiveThread() else { if (earliest_operation) - throw Exception("Operation timeout (no response) for path: " + earliest_operation->request->getPath(), ZOPERATIONTIMEOUT); + throw Exception("Operation timeout (no response) for path: " + earliest_operation->request->getPath(), Error::ZOPERATIONTIMEOUT); waited += max_wait; if (waited >= session_timeout.totalMicroseconds()) - throw Exception("Nothing is received in session timeout", ZOPERATIONTIMEOUT); + throw Exception("Nothing is received in session timeout", Error::ZOPERATIONTIMEOUT); } @@ -1193,7 +1200,7 @@ void ZooKeeper::receiveEvent() int32_t length; XID xid; int64_t zxid; - int32_t err; + Error err; read(length); size_t count_before_event = in->count(); @@ -1206,8 +1213,8 @@ void ZooKeeper::receiveEvent() if (xid == ping_xid) { - if (err) - throw Exception("Received error in heartbeat response: " + String(errorMessage(err)), ZRUNTIMEINCONSISTENCY); + if (err != Error::ZOK) + throw Exception("Received error in heartbeat response: " + String(errorMessage(err)), Error::ZRUNTIMEINCONSISTENCY); response = std::make_shared(); } @@ -1252,7 +1259,7 @@ void ZooKeeper::receiveEvent() auto it = operations.find(xid); if (it == operations.end()) - throw Exception("Received response for unknown xid", ZRUNTIMEINCONSISTENCY); + throw Exception("Received response for unknown xid", Error::ZRUNTIMEINCONSISTENCY); /// After this point, we must invoke callback, that we've grabbed from 'operations'. /// Invariant: all callbacks are invoked either in case of success or in case of error. @@ -1272,7 +1279,7 @@ void ZooKeeper::receiveEvent() if (!response) response = request_info.request->makeResponse(); - if (err) + if (err != Error::ZOK) response->error = err; else { @@ -1282,7 +1289,7 @@ void ZooKeeper::receiveEvent() int32_t actual_length = in->count() - count_before_event; if (length != actual_length) - throw Exception("Response length doesn't match. Expected: " + toString(length) + ", actual: " + toString(actual_length), ZMARSHALLINGERROR); + throw Exception("Response length doesn't match. Expected: " + toString(length) + ", actual: " + toString(actual_length), Error::ZMARSHALLINGERROR); } catch (...) { @@ -1294,7 +1301,7 @@ void ZooKeeper::receiveEvent() /// In case we cannot read the response, we should indicate it as the error of that type /// when the user cannot assume whether the request was processed or not. - response->error = ZCONNECTIONLOSS; + response->error = Error::ZCONNECTIONLOSS; if (request_info.callback) request_info.callback(*response); @@ -1361,8 +1368,8 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) ResponsePtr response = request_info.request->makeResponse(); response->error = request_info.request->probably_sent - ? ZCONNECTIONLOSS - : ZSESSIONEXPIRED; + ? Error::ZCONNECTIONLOSS + : Error::ZSESSIONEXPIRED; if (request_info.callback) { @@ -1390,7 +1397,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) WatchResponse response; response.type = SESSION; response.state = EXPIRED_SESSION; - response.error = ZSESSIONEXPIRED; + response.error = Error::ZSESSIONEXPIRED; for (auto & callback : path_watches.second) { @@ -1421,7 +1428,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) ResponsePtr response = info.request->makeResponse(); if (response) { - response->error = ZSESSIONEXPIRED; + response->error = Error::ZSESSIONEXPIRED; try { info.callback(*response); @@ -1437,7 +1444,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) WatchResponse response; response.type = SESSION; response.state = EXPIRED_SESSION; - response.error = ZSESSIONEXPIRED; + response.error = Error::ZSESSIONEXPIRED; try { info.watch(response); @@ -1466,9 +1473,9 @@ void ZooKeeper::pushRequest(RequestInfo && info) { info.request->xid = next_xid.fetch_add(1); if (info.request->xid == close_xid) - throw Exception("xid equal to close_xid", ZSESSIONEXPIRED); + throw Exception("xid equal to close_xid", Error::ZSESSIONEXPIRED); if (info.request->xid < 0) - throw Exception("XID overflow", ZSESSIONEXPIRED); + throw Exception("XID overflow", Error::ZSESSIONEXPIRED); } /// We must serialize 'pushRequest' and 'finalize' (from sendThread, receiveThread) calls @@ -1478,10 +1485,10 @@ void ZooKeeper::pushRequest(RequestInfo && info) std::lock_guard lock(push_request_mutex); if (expired) - throw Exception("Session expired", ZSESSIONEXPIRED); + throw Exception("Session expired", Error::ZSESSIONEXPIRED); if (!requests_queue.tryPush(std::move(info), operation_timeout.totalMilliseconds())) - throw Exception("Cannot push request to queue within operation timeout", ZOPERATIONTIMEOUT); + throw Exception("Cannot push request to queue within operation timeout", Error::ZOPERATIONTIMEOUT); } catch (...) { @@ -1651,7 +1658,7 @@ void ZooKeeper::close() request_info.request = std::make_shared(std::move(request)); if (!requests_queue.tryPush(std::move(request_info), operation_timeout.totalMilliseconds())) - throw Exception("Cannot push close request to queue within operation timeout", ZOPERATIONTIMEOUT); + throw Exception("Cannot push close request to queue within operation timeout", Error::ZOPERATIONTIMEOUT); ProfileEvents::increment(ProfileEvents::ZooKeeperClose); } diff --git a/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp b/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp index cd4c6e0a159..8440b4fe7c9 100644 --- a/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp +++ b/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp @@ -86,7 +86,7 @@ TEST(zkutil, MultiAsync) ops.clear(); auto res = fut.get(); - ASSERT_EQ(res.error, Coordination::ZOK); + ASSERT_EQ(res.error, Coordination::Error::ZOK); ASSERT_EQ(res.responses.size(), 2); } @@ -126,15 +126,15 @@ TEST(zkutil, MultiAsync) /// The test is quite heavy. It is normal if session is expired during this test. /// If we don't check that, the test will be flacky. - if (res.error != Coordination::ZSESSIONEXPIRED && res.error != Coordination::ZCONNECTIONLOSS) + if (res.error != Coordination::Error::ZSESSIONEXPIRED && res.error != Coordination::Error::ZCONNECTIONLOSS) { - ASSERT_EQ(res.error, Coordination::ZNODEEXISTS); + ASSERT_EQ(res.error, Coordination::Error::ZNODEEXISTS); ASSERT_EQ(res.responses.size(), 2); } } catch (const Coordination::Exception & e) { - if (e.code != Coordination::ZSESSIONEXPIRED && e.code != Coordination::ZCONNECTIONLOSS) + if (e.code != Coordination::Error::ZSESSIONEXPIRED && e.code != Coordination::Error::ZCONNECTIONLOSS) throw; } } diff --git a/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp b/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp index d245428db8e..e09c72a4d6c 100644 --- a/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp +++ b/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp @@ -39,12 +39,12 @@ int main(int argc, char ** argv) ops.emplace_back(zkutil::makeRemoveRequest("/test/zk_expiration_test", -1)); Coordination::Responses responses; - int32_t code = zk.tryMultiNoThrow(ops, responses); + Coordination::Error code = zk.tryMultiNoThrow(ops, responses); - std::cout << time(nullptr) - time0 << "s: " << zkutil::ZooKeeper::error2string(code) << std::endl; + std::cout << time(nullptr) - time0 << "s: " << Coordination::errorMessage(code) << std::endl; try { - if (code) + if (code != Coordination::Error::ZOK) std::cout << "Path: " << zkutil::KeeperMultiException(code, ops, responses).getPathForFirstFailedOp() << std::endl; } catch (...) diff --git a/src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp b/src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp index d9d3402fa32..89659fa5e46 100644 --- a/src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp +++ b/src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp @@ -49,8 +49,8 @@ try zk.create("/test", "old", false, false, {}, [&](const CreateResponse & response) { - if (response.error) - std::cerr << "Error (create) " << response.error << ": " << errorMessage(response.error) << '\n'; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Error (create): " << errorMessage(response.error) << '\n'; else std::cerr << "Created path: " << response.path_created << '\n'; @@ -64,8 +64,8 @@ try zk.get("/test", [&](const GetResponse & response) { - if (response.error) - std::cerr << "Error (get) " << response.error << ": " << errorMessage(response.error) << '\n'; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Error (get): " << errorMessage(response.error) << '\n'; else std::cerr << "Value: " << response.data << '\n'; @@ -73,8 +73,8 @@ try }, [](const WatchResponse & response) { - if (response.error) - std::cerr << "Watch (get) on /test, Error " << response.error << ": " << errorMessage(response.error) << '\n'; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Watch (get) on /test, Error: " << errorMessage(response.error) << '\n'; else std::cerr << "Watch (get) on /test, path: " << response.path << ", type: " << response.type << '\n'; }); @@ -86,8 +86,8 @@ try zk.set("/test", "new", -1, [&](const SetResponse & response) { - if (response.error) - std::cerr << "Error (set) " << response.error << ": " << errorMessage(response.error) << '\n'; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Error (set): " << errorMessage(response.error) << '\n'; else std::cerr << "Set\n"; @@ -101,8 +101,8 @@ try zk.list("/", [&](const ListResponse & response) { - if (response.error) - std::cerr << "Error (list) " << response.error << ": " << errorMessage(response.error) << '\n'; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Error (list): " << errorMessage(response.error) << '\n'; else { std::cerr << "Children:\n"; @@ -114,8 +114,8 @@ try }, [](const WatchResponse & response) { - if (response.error) - std::cerr << "Watch (list) on /, Error " << response.error << ": " << errorMessage(response.error) << '\n'; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Watch (list) on /, Error: " << errorMessage(response.error) << '\n'; else std::cerr << "Watch (list) on /, path: " << response.path << ", type: " << response.type << '\n'; }); @@ -127,8 +127,8 @@ try zk.exists("/test", [&](const ExistsResponse & response) { - if (response.error) - std::cerr << "Error (exists) " << response.error << ": " << errorMessage(response.error) << '\n'; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Error (exists): " << errorMessage(response.error) << '\n'; else std::cerr << "Exists\n"; @@ -136,8 +136,8 @@ try }, [](const WatchResponse & response) { - if (response.error) - std::cerr << "Watch (exists) on /test, Error " << response.error << ": " << errorMessage(response.error) << '\n'; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Watch (exists) on /test, Error: " << errorMessage(response.error) << '\n'; else std::cerr << "Watch (exists) on /test, path: " << response.path << ", type: " << response.type << '\n'; }); @@ -148,8 +148,8 @@ try zk.remove("/test", -1, [&](const RemoveResponse & response) { - if (response.error) - std::cerr << "Error (remove) " << response.error << ": " << errorMessage(response.error) << '\n'; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Error (remove): " << errorMessage(response.error) << '\n'; else std::cerr << "Removed\n"; @@ -184,13 +184,13 @@ try zk.multi(ops, [&](const MultiResponse & response) { - if (response.error) - std::cerr << "Error (multi) " << response.error << ": " << errorMessage(response.error) << '\n'; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Error (multi): " << errorMessage(response.error) << '\n'; else { for (const auto & elem : response.responses) - if (elem->error) - std::cerr << "Error (elem) " << elem->error << ": " << errorMessage(elem->error) << '\n'; + if (elem->error != Coordination::Error::ZOK) + std::cerr << "Error (elem): " << errorMessage(elem->error) << '\n'; std::cerr << "Created path: " << dynamic_cast(*response.responses[0]).path_created << '\n'; } diff --git a/src/Common/ZooKeeper/tests/zookeeper_impl.cpp b/src/Common/ZooKeeper/tests/zookeeper_impl.cpp index 74ba63514f2..3c2e52c93f2 100644 --- a/src/Common/ZooKeeper/tests/zookeeper_impl.cpp +++ b/src/Common/ZooKeeper/tests/zookeeper_impl.cpp @@ -9,8 +9,8 @@ try zookeeper.create("/test", "hello", false, false, {}, [](const Coordination::CreateResponse & response) { - if (response.error) - std::cerr << "Error " << response.error << ": " << Coordination::errorMessage(response.error) << "\n"; + if (response.error != Coordination::Error::ZOK) + std::cerr << "Error: " << Coordination::errorMessage(response.error) << "\n"; else std::cerr << "Path created: " << response.path_created << "\n"; }); diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index dac51b21081..9f89fa9199b 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -422,7 +422,7 @@ void DDLWorker::processTasks() } catch (const Coordination::Exception & e) { - if (server_startup && e.code == Coordination::ZNONODE) + if (server_startup && e.code == Coordination::Error::ZNONODE) { LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true)); } @@ -603,15 +603,15 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) auto code = zookeeper->tryCreate(active_node_path, "", zkutil::CreateMode::Ephemeral, dummy); - if (code == Coordination::ZOK || code == Coordination::ZNODEEXISTS) + if (code == Coordination::Error::ZOK || code == Coordination::Error::ZNODEEXISTS) { // Ok } - else if (code == Coordination::ZNONODE) + else if (code == Coordination::Error::ZNONODE) { /// There is no parent createStatusDirs(task.entry_path, zookeeper); - if (Coordination::ZOK != zookeeper->tryCreate(active_node_path, "", zkutil::CreateMode::Ephemeral, dummy)) + if (Coordination::Error::ZOK != zookeeper->tryCreate(active_node_path, "", zkutil::CreateMode::Ephemeral, dummy)) throw Coordination::Exception(code, active_node_path); } else @@ -915,8 +915,9 @@ void DDLWorker::createStatusDirs(const std::string & node_path, const ZooKeeperP ops.emplace_back(std::make_shared(std::move(request))); } Coordination::Responses responses; - int code = zookeeper->tryMulti(ops, responses); - if (code && code != Coordination::ZNODEEXISTS) + Coordination::Error code = zookeeper->tryMulti(ops, responses); + if (code != Coordination::Error::ZOK + && code != Coordination::Error::ZNODEEXISTS) throw Coordination::Exception(code); } @@ -1013,7 +1014,7 @@ void DDLWorker::runMainThread() } } } - else if (e.code == Coordination::ZNONODE) + else if (e.code == Coordination::Error::ZNONODE) { LOG_ERROR(log, "ZooKeeper error: {}", getCurrentExceptionMessage(true)); } @@ -1201,8 +1202,8 @@ private: static Strings getChildrenAllowNoNode(const std::shared_ptr & zookeeper, const String & node_path) { Strings res; - int code = zookeeper->tryGetChildren(node_path, res); - if (code && code != Coordination::ZNONODE) + Coordination::Error code = zookeeper->tryGetChildren(node_path, res); + if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE) throw Coordination::Exception(code, node_path); return res; } diff --git a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp index 762dbc7d5b6..6b00215fd26 100644 --- a/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp +++ b/src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp @@ -104,13 +104,13 @@ EphemeralLocksInAllPartitions::EphemeralLocksInAllPartitions( lock_ops.push_back(zkutil::makeCheckRequest(block_numbers_path, partitions_stat.version)); Coordination::Responses lock_responses; - int rc = zookeeper.tryMulti(lock_ops, lock_responses); - if (rc == Coordination::ZBADVERSION) + Coordination::Error rc = zookeeper.tryMulti(lock_ops, lock_responses); + if (rc == Coordination::Error::ZBADVERSION) { LOG_TRACE(&Poco::Logger::get("EphemeralLocksInAllPartitions"), "Someone has inserted a block in a new partition while we were creating locks. Retry."); continue; } - else if (rc != Coordination::ZOK) + else if (rc != Coordination::Error::ZOK) throw Coordination::Exception(rc); for (size_t i = 0; i < partitions.size(); ++i) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index d8c1103809f..c67ea11f56f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -85,7 +85,7 @@ void ReplicatedMergeTreeBlockOutputStream::checkQuorumPrecondition(zkutil::ZooKe */ auto quorum_status = quorum_status_future.get(); - if (quorum_status.error != Coordination::ZNONODE) + if (quorum_status.error != Coordination::Error::ZNONODE) throw Exception("Quorum for previous write has not been satisfied yet. Status: " + quorum_status.data, ErrorCodes::UNSATISFIED_QUORUM_FOR_PREVIOUS_WRITE); /// Both checks are implicitly made also later (otherwise there would be a race condition). @@ -93,7 +93,7 @@ void ReplicatedMergeTreeBlockOutputStream::checkQuorumPrecondition(zkutil::ZooKe auto is_active = is_active_future.get(); auto host = host_future.get(); - if (is_active.error == Coordination::ZNONODE || host.error == Coordination::ZNONODE) + if (is_active.error == Coordination::Error::ZNONODE || host.error == Coordination::Error::ZNONODE) throw Exception("Replica is not active right now", ErrorCodes::READONLY); quorum_info.is_active_node_value = is_active.data; @@ -299,9 +299,9 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo storage.renameTempPartAndAdd(part, nullptr, &transaction); Coordination::Responses responses; - int32_t multi_code = zookeeper->tryMultiNoThrow(ops, responses); /// 1 RTT + Coordination::Error multi_code = zookeeper->tryMultiNoThrow(ops, responses); /// 1 RTT - if (multi_code == Coordination::ZOK) + if (multi_code == Coordination::Error::ZOK) { transaction.commit(); storage.merge_selecting_task->schedule(); @@ -309,8 +309,8 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo /// Lock nodes have been already deleted, do not delete them in destructor block_number_lock->assumeUnlocked(); } - else if (multi_code == Coordination::ZCONNECTIONLOSS - || multi_code == Coordination::ZOPERATIONTIMEOUT) + else if (multi_code == Coordination::Error::ZCONNECTIONLOSS + || multi_code == Coordination::Error::ZOPERATIONTIMEOUT) { /** If the connection is lost, and we do not know if the changes were applied, we can not delete the local part * if the changes were applied, the inserted block appeared in `/blocks/`, and it can not be inserted again. @@ -326,7 +326,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo { String failed_op_path = zkutil::KeeperMultiException(multi_code, ops, responses).getPathForFirstFailedOp(); - if (multi_code == Coordination::ZNODEEXISTS && deduplicate_block && failed_op_path == block_id_path) + if (multi_code == Coordination::Error::ZNODEEXISTS && deduplicate_block && failed_op_path == block_id_path) { /// Block with the same id have just appeared in table (or other replica), rollback thee insertion. LOG_INFO(log, "Block with ID {} already exists; ignoring it (removing part {})", block_id, part->name); @@ -336,7 +336,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo last_block_is_duplicate = true; ProfileEvents::increment(ProfileEvents::DuplicatedInsertedBlocks); } - else if (multi_code == Coordination::ZNODEEXISTS && failed_op_path == quorum_info.status_path) + else if (multi_code == Coordination::Error::ZNODEEXISTS && failed_op_path == quorum_info.status_path) { transaction.rollback(); @@ -347,7 +347,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo /// NOTE: We could be here if the node with the quorum existed, but was quickly removed. transaction.rollback(); throw Exception("Unexpected logical error while adding block " + toString(block_number) + " with ID '" + block_id + "': " - + zkutil::ZooKeeper::error2string(multi_code) + ", path " + failed_op_path, + + Coordination::errorMessage(multi_code) + ", path " + failed_op_path, ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR); } } @@ -355,13 +355,13 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo { transaction.rollback(); throw Exception("Unrecoverable network error while adding block " + toString(block_number) + " with ID '" + block_id + "': " - + zkutil::ZooKeeper::error2string(multi_code), ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR); + + Coordination::errorMessage(multi_code), ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR); } else { transaction.rollback(); throw Exception("Unexpected ZooKeeper error while adding block " + toString(block_number) + " with ID '" + block_id + "': " - + zkutil::ZooKeeper::error2string(multi_code), ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR); + + Coordination::errorMessage(multi_code), ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR); } if (quorum) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index de91a5d5940..1bc132eaba4 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -40,7 +40,7 @@ void ReplicatedMergeTreeCleanupThread::run() { tryLogCurrentException(log, __PRETTY_FUNCTION__); - if (e.code == Coordination::ZSESSIONEXPIRED) + if (e.code == Coordination::Error::ZSESSIONEXPIRED) return; } catch (...) @@ -319,15 +319,15 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() for (auto & pair : try_remove_futures) { const String & path = pair.first; - int32_t rc = pair.second.get().error; - if (rc == Coordination::ZNOTEMPTY) + Coordination::Error rc = pair.second.get().error; + if (rc == Coordination::Error::ZNOTEMPTY) { /// Can happen if there are leftover block nodes with children created by previous server versions. zookeeper->removeRecursive(path); cached_block_stats.erase(first_outdated_block->node); } - else if (rc) - LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, zkutil::ZooKeeper::error2string(rc)); + else if (rc != Coordination::Error::ZOK) + LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, Coordination::errorMessage(rc)); else { /// Successfully removed blocks have to be removed from cache @@ -348,7 +348,7 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper & Strings blocks; Coordination::Stat stat; - if (zookeeper.tryGetChildren(storage.zookeeper_path + "/blocks", blocks, &stat)) + if (Coordination::Error::ZOK != zookeeper.tryGetChildren(storage.zookeeper_path + "/blocks", blocks, &stat)) throw Exception(storage.zookeeper_path + "/blocks doesn't exist", ErrorCodes::NOT_FOUND_NODE); /// Seems like this code is obsolete, because we delete blocks from cache @@ -391,7 +391,7 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper & for (auto & elem : exists_futures) { auto status = elem.second.get(); - if (status.error != Coordination::ZNONODE) + if (status.error != Coordination::Error::ZNONODE) { cached_block_stats.emplace(elem.first, status.stat.ctime); timed_blocks.emplace_back(elem.first, status.stat.ctime); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 8f99f315620..0d824fa2dd8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -368,7 +368,7 @@ void ReplicatedMergeTreePartCheckThread::run() { tryLogCurrentException(log, __PRETTY_FUNCTION__); - if (e.code == Coordination::ZSESSIONEXPIRED) + if (e.code == Coordination::Error::ZSESSIONEXPIRED) return; task->scheduleAfter(PART_CHECK_ERROR_SLEEP_MS); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index a6dec4816bf..09e9cd494ca 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -319,8 +319,8 @@ void ReplicatedMergeTreeQueue::updateTimesInZooKeeper( Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); - if (code) - LOG_ERROR(log, "Couldn't set value of nodes for insert times ({}/min_unprocessed_insert_time, max_processed_insert_time): {}", replica_path, zkutil::ZooKeeper::error2string(code) + ". This shouldn't happen often."); + if (code != Coordination::Error::ZOK) + LOG_ERROR(log, "Couldn't set value of nodes for insert times ({}/min_unprocessed_insert_time, max_processed_insert_time): {}. This shouldn't happen often.", replica_path, Coordination::errorMessage(code)); } } @@ -364,8 +364,8 @@ void ReplicatedMergeTreeQueue::removeProcessedEntry(zkutil::ZooKeeperPtr zookeep notifySubscribers(queue_size); auto code = zookeeper->tryRemove(replica_path + "/queue/" + entry->znode_name); - if (code) - LOG_ERROR(log, "Couldn't remove {}/queue/{}: {}. This shouldn't happen often.", replica_path, entry->znode_name, zkutil::ZooKeeper::error2string(code)); + if (code != Coordination::Error::ZOK) + LOG_ERROR(log, "Couldn't remove {}/queue/{}: {}. This shouldn't happen often.", replica_path, entry->znode_name, Coordination::errorMessage(code)); updateTimesInZooKeeper(zookeeper, min_unprocessed_insert_time_changed, max_processed_insert_time_changed); } @@ -720,7 +720,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( std::lock_guard lock(update_mutations_mutex); auto rc = zookeeper->tryRemove(zookeeper_path + "/mutations/" + mutation_id); - if (rc == Coordination::ZOK) + if (rc == Coordination::Error::ZOK) LOG_DEBUG(log, "Removed mutation {} from ZooKeeper.", mutation_id); ReplicatedMergeTreeMutationEntryPtr entry; @@ -844,8 +844,8 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( if ((*it)->currently_executing) to_wait.push_back(*it); auto code = zookeeper->tryRemove(replica_path + "/queue/" + (*it)->znode_name); - if (code) - LOG_INFO(log, "Couldn't remove {}: {}", replica_path + "/queue/" + (*it)->znode_name, zkutil::ZooKeeper::error2string(code)); + if (code != Coordination::Error::ZOK) + LOG_INFO(log, "Couldn't remove {}: {}", replica_path + "/queue/" + (*it)->znode_name, Coordination::errorMessage(code)); updateStateOnQueueEntryRemoval( *it, /* is_successful = */ false, @@ -1625,7 +1625,7 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( for (auto & block : block_infos) { Coordination::GetResponse resp = block.contents_future.get(); - if (!resp.error && lock_holder_paths.count(resp.data)) + if (resp.error == Coordination::Error::ZOK && lock_holder_paths.count(resp.data)) committing_blocks[block.partition].insert(block.number); } } @@ -1633,7 +1633,7 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( queue_.pullLogsToQueue(zookeeper); Coordination::GetResponse quorum_status_response = quorum_status_future.get(); - if (!quorum_status_response.error) + if (quorum_status_response.error == Coordination::Error::ZOK) { ReplicatedMergeTreeQuorumEntry quorum_status; quorum_status.fromString(quorum_status_response.data); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 93d652f2be0..0ec9b8069ac 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -234,7 +234,7 @@ void ReplicatedMergeTreeRestartingThread::removeFailedQuorumParts() auto zookeeper = storage.getZooKeeper(); Strings failed_parts; - if (zookeeper->tryGetChildren(storage.zookeeper_path + "/quorum/failed_parts", failed_parts) != Coordination::ZOK) + if (zookeeper->tryGetChildren(storage.zookeeper_path + "/quorum/failed_parts", failed_parts) != Coordination::Error::ZOK) return; /// Firstly, remove parts from ZooKeeper @@ -294,12 +294,12 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() { auto code = zookeeper->tryRemove(is_active_path, stat.version); - if (code == Coordination::ZBADVERSION) + if (code == Coordination::Error::ZBADVERSION) throw Exception("Another instance of replica " + storage.replica_path + " was created just now." " You shouldn't run multiple instances of same replica. You need to check configuration files.", ErrorCodes::REPLICA_IS_ALREADY_ACTIVE); - if (code && code != Coordination::ZNONODE) + if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE) throw Coordination::Exception(code, is_active_path); } @@ -314,7 +314,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() } catch (const Coordination::Exception & e) { - if (e.code == Coordination::ZNODEEXISTS) + if (e.code == Coordination::Error::ZNODEEXISTS) throw Exception("Replica " + storage.replica_path + " appears to be already active. If you're sure it's not, " "try again in a minute or remove znode " + storage.replica_path + "/is_active manually", ErrorCodes::REPLICA_IS_ALREADY_ACTIVE); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 809536f7452..e7d9e214995 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -441,8 +441,8 @@ bool StorageReplicatedMergeTree::createTableIfNotExists() LOG_WARNING(log, "Removing leftovers from table {} (this might take several minutes)", zookeeper_path); Strings children; - int32_t code = zookeeper->tryGetChildren(zookeeper_path, children); - if (code == Coordination::ZNONODE) + Coordination::Error code = zookeeper->tryGetChildren(zookeeper_path, children); + if (code == Coordination::Error::ZNONODE) { LOG_WARNING(log, "Table {} is already finished removing by another replica right now", replica_path); } @@ -458,16 +458,16 @@ bool StorageReplicatedMergeTree::createTableIfNotExists() ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path, -1)); code = zookeeper->tryMulti(ops, responses); - if (code == Coordination::ZNONODE) + if (code == Coordination::Error::ZNONODE) { LOG_WARNING(log, "Table {} is already finished removing by another replica right now", replica_path); } - else if (code == Coordination::ZNOTEMPTY) + else if (code == Coordination::Error::ZNOTEMPTY) { throw Exception(fmt::format( "The old table was not completely removed from ZooKeeper, {} still exists and may contain some garbage. But it should never happen according to the logic of operations (it's a bug).", zookeeper_path), ErrorCodes::LOGICAL_ERROR); } - else if (code != Coordination::ZOK) + else if (code != Coordination::Error::ZOK) { /// It is still possible that ZooKeeper session is expired or server is killed in the middle of the delete operation. zkutil::KeeperMultiException::check(code, ops, responses); @@ -535,12 +535,12 @@ bool StorageReplicatedMergeTree::createTableIfNotExists() Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); - if (code == Coordination::ZNODEEXISTS) + if (code == Coordination::Error::ZNODEEXISTS) { LOG_WARNING(log, "It looks like the table {} was created by another server at the same moment, will retry", zookeeper_path); continue; } - else if (code != Coordination::ZOK) + else if (code != Coordination::Error::ZOK) { zkutil::KeeperMultiException::check(code, ops, responses); } @@ -557,7 +557,7 @@ void StorageReplicatedMergeTree::createReplica() LOG_DEBUG(log, "Creating replica {}", replica_path); - int32_t code; + Coordination::Error code; do { @@ -599,15 +599,15 @@ void StorageReplicatedMergeTree::createReplica() Coordination::Responses responses; code = zookeeper->tryMulti(ops, responses); - if (code == Coordination::ZNODEEXISTS) + if (code == Coordination::Error::ZNODEEXISTS) { throw Exception("Replica " + replica_path + " already exists.", ErrorCodes::REPLICA_IS_ALREADY_EXIST); } - else if (code == Coordination::ZBADVERSION) + else if (code == Coordination::Error::ZBADVERSION) { LOG_ERROR(log, "Retrying createReplica(), because some other replicas were created at the same time"); } - else if (code == Coordination::ZNONODE) + else if (code == Coordination::Error::ZNONODE) { throw Exception("Table " + zookeeper_path + " was suddenly removed.", ErrorCodes::ALL_REPLICAS_LOST); } @@ -615,7 +615,7 @@ void StorageReplicatedMergeTree::createReplica() { zkutil::KeeperMultiException::check(code, ops, responses); } - } while (code == Coordination::ZBADVERSION); + } while (code == Coordination::Error::ZBADVERSION); } void StorageReplicatedMergeTree::drop() @@ -640,7 +640,7 @@ void StorageReplicatedMergeTree::drop() /// Check that `zookeeper_path` exists: it could have been deleted by another replica after execution of previous line. Strings replicas; - if (Coordination::ZOK == zookeeper->tryGetChildren(zookeeper_path + "/replicas", replicas) && replicas.empty()) + if (Coordination::Error::ZOK == zookeeper->tryGetChildren(zookeeper_path + "/replicas", replicas) && replicas.empty()) { LOG_INFO(log, "{} is the last replica, will remove table", replica_path); @@ -656,17 +656,17 @@ void StorageReplicatedMergeTree::drop() Coordination::Responses responses; ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path + "/replicas", -1)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/dropped", "", zkutil::CreateMode::Persistent)); - int32_t code = zookeeper->tryMulti(ops, responses); + Coordination::Error code = zookeeper->tryMulti(ops, responses); - if (code == Coordination::ZNONODE || code == Coordination::ZNODEEXISTS) + if (code == Coordination::Error::ZNONODE || code == Coordination::Error::ZNODEEXISTS) { LOG_WARNING(log, "Table {} is already started to be removing by another replica right now", replica_path); } - else if (code == Coordination::ZNOTEMPTY) + else if (code == Coordination::Error::ZNOTEMPTY) { LOG_WARNING(log, "Another replica was suddenly created, will keep the table {}", replica_path); } - else if (code != Coordination::ZOK) + else if (code != Coordination::Error::ZOK) { zkutil::KeeperMultiException::check(code, ops, responses); } @@ -676,7 +676,7 @@ void StorageReplicatedMergeTree::drop() Strings children; code = zookeeper->tryGetChildren(zookeeper_path, children); - if (code == Coordination::ZNONODE) + if (code == Coordination::Error::ZNONODE) { LOG_WARNING(log, "Table {} is already finished removing by another replica right now", replica_path); } @@ -692,16 +692,16 @@ void StorageReplicatedMergeTree::drop() ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path, -1)); code = zookeeper->tryMulti(ops, responses); - if (code == Coordination::ZNONODE) + if (code == Coordination::Error::ZNONODE) { LOG_WARNING(log, "Table {} is already finished removing by another replica right now", replica_path); } - else if (code == Coordination::ZNOTEMPTY) + else if (code == Coordination::Error::ZNOTEMPTY) { LOG_ERROR(log, "Table was not completely removed from ZooKeeper, {} still exists and may contain some garbage.", zookeeper_path); } - else if (code != Coordination::ZOK) + else if (code != Coordination::Error::ZOK) { /// It is still possible that ZooKeeper session is expired or server is killed in the middle of the delete operation. zkutil::KeeperMultiException::check(code, ops, responses); @@ -936,7 +936,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) time_t part_create_time = 0; Coordination::ExistsResponse exists_resp = exists_futures[i].get(); - if (!exists_resp.error) + if (exists_resp.error == Coordination::Error::ZOK) { part_create_time = exists_resp.stat.ctime / 1000; removePartFromZooKeeper(part_name, ops, exists_resp.stat.numChildren > 0); @@ -1107,7 +1107,7 @@ MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAnd size_t num_check_ops = 2 * absent_part_paths_on_replicas.size(); size_t failed_op_index = e.failed_op_index; - if (failed_op_index < num_check_ops && e.code == Coordination::ZNODEEXISTS) + if (failed_op_index < num_check_ops && e.code == Coordination::Error::ZNODEEXISTS) { LOG_INFO(log, "The part {} on a replica suddenly appeared, will recheck checksums", e.getPathForFirstFailedOp()); } @@ -1584,15 +1584,15 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); - if (code == Coordination::ZOK) + if (code == Coordination::Error::ZOK) { LOG_DEBUG(log, "Marked quorum for part {} as failed.", entry.new_part_name); queue.removeFromVirtualParts(part_info); return true; } - else if (code == Coordination::ZBADVERSION || code == Coordination::ZNONODE || code == Coordination::ZNODEEXISTS) + else if (code == Coordination::Error::ZBADVERSION || code == Coordination::Error::ZNONODE || code == Coordination::Error::ZNODEEXISTS) { - LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", entry.new_part_name, zkutil::ZooKeeper::error2string(code)); + LOG_DEBUG(log, "State was changed or isn't expected when trying to mark quorum for part {} as failed. Code: {}", entry.new_part_name, Coordination::errorMessage(code)); } else throw Coordination::Exception(code); @@ -2088,7 +2088,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo auto rc = zookeeper->tryMulti(ops, responses); - if (rc == Coordination::ZOK) + if (rc == Coordination::Error::ZOK) { break; } @@ -2256,7 +2256,7 @@ void StorageReplicatedMergeTree::queueUpdatingTask() { tryLogCurrentException(log, __PRETTY_FUNCTION__); - if (e.code == Coordination::ZSESSIONEXPIRED) + if (e.code == Coordination::Error::ZSESSIONEXPIRED) { restarting_thread.wakeup(); return; @@ -2282,7 +2282,7 @@ void StorageReplicatedMergeTree::mutationsUpdatingTask() { tryLogCurrentException(log, __PRETTY_FUNCTION__); - if (e.code == Coordination::ZSESSIONEXPIRED) + if (e.code == Coordination::Error::ZSESSIONEXPIRED) return; mutations_updating_task->scheduleAfter(QUEUE_UPDATE_ERROR_SLEEP_MS); @@ -2525,7 +2525,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( for (size_t i = 0; i < parts.size(); ++i) { /// If there is no information about part in ZK, we will not merge it. - if (exists_futures[i].get().error == Coordination::ZNONODE) + if (exists_futures[i].get().error == Coordination::Error::ZNONODE) { all_in_zk = false; @@ -2871,16 +2871,16 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name) ops.emplace_back(zkutil::makeSetRequest(quorum_last_part_path, new_added_parts, added_parts_stat.version)); auto code = zookeeper->tryMulti(ops, responses); - if (code == Coordination::ZOK) + if (code == Coordination::Error::ZOK) { break; } - else if (code == Coordination::ZNONODE) + else if (code == Coordination::Error::ZNONODE) { /// The quorum has already been achieved. break; } - else if (code == Coordination::ZBADVERSION) + else if (code == Coordination::Error::ZBADVERSION) { /// Node was updated meanwhile. We must re-read it and repeat all the actions. continue; @@ -2893,16 +2893,16 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name) /// We update the node, registering there one more replica. auto code = zookeeper->trySet(quorum_status_path, quorum_entry.toString(), stat.version); - if (code == Coordination::ZOK) + if (code == Coordination::Error::ZOK) { break; } - else if (code == Coordination::ZNONODE) + else if (code == Coordination::Error::ZNONODE) { /// The quorum has already been achieved. break; } - else if (code == Coordination::ZBADVERSION) + else if (code == Coordination::Error::ZBADVERSION) { /// Node was updated meanwhile. We must re-read it and repeat all the actions. continue; @@ -2946,16 +2946,16 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) auto code = zookeeper->trySet(quorum_last_part_path, new_added_parts, added_parts_stat.version); - if (code == Coordination::ZOK) + if (code == Coordination::Error::ZOK) { break; } - else if (code == Coordination::ZNONODE) + else if (code == Coordination::Error::ZNONODE) { /// Node is deleted. It is impossible, but it is Ok. break; } - else if (code == Coordination::ZBADVERSION) + else if (code == Coordination::Error::ZBADVERSION) { /// Node was updated meanwhile. We must re-read it and repeat all the actions. continue; @@ -3643,9 +3643,9 @@ void StorageReplicatedMergeTree::alter( } Coordination::Responses results; - int32_t rc = zookeeper->tryMulti(ops, results); + Coordination::Error rc = zookeeper->tryMulti(ops, results); - if (rc == Coordination::ZOK) + if (rc == Coordination::Error::ZOK) { if (alter_entry->have_mutation) { @@ -3665,9 +3665,9 @@ void StorageReplicatedMergeTree::alter( } break; } - else if (rc == Coordination::ZBADVERSION) + else if (rc == Coordination::Error::ZBADVERSION) { - if (results[0]->error) + if (results[0]->error != Coordination::Error::ZOK) throw Exception("Metadata on replica is not up to date with common metadata in Zookeeper. Cannot alter", ErrorCodes::CANNOT_ASSIGN_ALTER); continue; @@ -3987,8 +3987,8 @@ StorageReplicatedMergeTree::allocateBlockNumber( ops.push_back(zkutil::makeSetRequest(block_numbers_path, "", -1)); Coordination::Responses responses; - int code = zookeeper->tryMulti(ops, responses); - if (code && code != Coordination::ZNODEEXISTS) + Coordination::Error code = zookeeper->tryMulti(ops, responses); + if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNODEEXISTS) zkutil::KeeperMultiException::check(code, ops, responses); } @@ -4001,7 +4001,7 @@ StorageReplicatedMergeTree::allocateBlockNumber( } catch (const zkutil::KeeperMultiException & e) { - if (e.code == Coordination::ZNODEEXISTS && e.getPathForFirstFailedOp() == zookeeper_block_id_path) + if (e.code == Coordination::Error::ZNODEEXISTS && e.getPathForFirstFailedOp() == zookeeper_block_id_path) return {}; throw Exception("Cannot allocate block number in ZooKeeper: " + e.displayText(), ErrorCodes::KEEPER_EXCEPTION); @@ -4690,9 +4690,9 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const mutations_path + "/", entry.toString(), zkutil::CreateMode::PersistentSequential)); Coordination::Responses responses; - int32_t rc = zookeeper->tryMulti(requests, responses); + Coordination::Error rc = zookeeper->tryMulti(requests, responses); - if (rc == Coordination::ZOK) + if (rc == Coordination::Error::ZOK) { const String & path_created = dynamic_cast(responses[1].get())->path_created; @@ -4700,7 +4700,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const LOG_TRACE(log, "Created mutation with ID {}", entry.znode_name); break; } - else if (rc == Coordination::ZBADVERSION) + else if (rc == Coordination::Error::ZBADVERSION) { LOG_TRACE(log, "Version conflict when trying to create a mutation node, retrying..."); continue; @@ -4892,7 +4892,7 @@ bool StorageReplicatedMergeTree::tryRemovePartsFromZooKeeperWithRetries(const St for (size_t i = 0; i < part_names.size(); ++i) { Coordination::ExistsResponse exists_resp = exists_futures[i].get(); - if (!exists_resp.error) + if (exists_resp.error == Coordination::Error::ZOK) { Coordination::Requests ops; removePartFromZooKeeper(part_names[i], ops, exists_resp.stat.numChildren > 0); @@ -4904,7 +4904,7 @@ bool StorageReplicatedMergeTree::tryRemovePartsFromZooKeeperWithRetries(const St { auto response = future.get(); - if (response.error == 0 || response.error == Coordination::ZNONODE) + if (response.error == Coordination::Error::ZOK || response.error == Coordination::Error::ZNONODE) continue; if (Coordination::isHardwareError(response.error)) @@ -4953,7 +4953,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( for (size_t i = 0; i < part_names.size(); ++i) { Coordination::ExistsResponse exists_resp = exists_futures[i].get(); - if (!exists_resp.error) + if (exists_resp.error == Coordination::Error::ZOK) { Coordination::Requests ops; removePartFromZooKeeper(part_names[i], ops, exists_resp.stat.numChildren > 0); @@ -4982,9 +4982,9 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( continue; auto response = future.get(); - if (response.error == Coordination::ZOK) + if (response.error == Coordination::Error::ZOK) continue; - else if (response.error == Coordination::ZNONODE) + else if (response.error == Coordination::Error::ZNONODE) { LOG_DEBUG(log, "There is no part {} in ZooKeeper, it was only in filesystem", part_names[i]); continue; @@ -4996,7 +4996,7 @@ void StorageReplicatedMergeTree::removePartsFromZooKeeper( continue; } else - LOG_WARNING(log, "Cannot remove part {} from ZooKeeper: {}", part_names[i], zkutil::ZooKeeper::error2string(response.error)); + LOG_WARNING(log, "Cannot remove part {} from ZooKeeper: {}", part_names[i], Coordination::errorMessage(response.error)); } } @@ -5005,7 +5005,7 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( zkutil::ZooKeeper & zookeeper, const String & partition_id, Int64 min_block_num, Int64 max_block_num) { Strings blocks; - if (zookeeper.tryGetChildren(zookeeper_path + "/blocks", blocks)) + if (Coordination::Error::ZOK != zookeeper.tryGetChildren(zookeeper_path + "/blocks", blocks)) throw Exception(zookeeper_path + "/blocks doesn't exist", ErrorCodes::NOT_FOUND_NODE); String partition_prefix = partition_id + "_"; @@ -5025,7 +5025,7 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( const String & path = pair.first; auto result = pair.second.get(); - if (result.error == Coordination::ZNONODE) + if (result.error == Coordination::Error::ZNONODE) continue; ReadBufferFromString buf(result.data); @@ -5038,14 +5038,14 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( for (auto & pair : to_delete_futures) { const String & path = pair.first; - int32_t rc = pair.second.get().error; - if (rc == Coordination::ZNOTEMPTY) + Coordination::Error rc = pair.second.get().error; + if (rc == Coordination::Error::ZNOTEMPTY) { /// Can happen if there are leftover block nodes with children created by previous server versions. zookeeper.removeRecursive(path); } - else if (rc) - LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, zkutil::ZooKeeper::error2string(rc)); + else if (rc != Coordination::Error::ZOK) + LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, Coordination::errorMessage(rc)); } LOG_TRACE(log, "Deleted {} deduplication block IDs in partition ID {}", to_delete_futures.size(), partition_id); diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp index c3f1d8a8505..d5806667d1f 100644 --- a/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/src/Storages/System/StorageSystemZooKeeper.cpp @@ -131,7 +131,7 @@ void StorageSystemZooKeeper::fillData(MutableColumns & res_columns, const Contex for (size_t i = 0, size = nodes.size(); i < size; ++i) { auto res = futures[i].get(); - if (res.error == Coordination::ZNONODE) + if (res.error == Coordination::Error::ZNONODE) continue; /// Node was deleted meanwhile. const Coordination::Stat & stat = res.stat; diff --git a/src/Storages/tests/get_current_inserts_in_replicated.cpp b/src/Storages/tests/get_current_inserts_in_replicated.cpp index aba69684045..f1a87b5ca05 100644 --- a/src/Storages/tests/get_current_inserts_in_replicated.cpp +++ b/src/Storages/tests/get_current_inserts_in_replicated.cpp @@ -86,7 +86,7 @@ try for (BlockInfo & block : block_infos) { Coordination::GetResponse resp = block.contents_future.get(); - if (!resp.error && lock_holder_paths.count(resp.data)) + if (resp.error == Coordination::Error::ZOK && lock_holder_paths.count(resp.data)) { ++total_count; current_inserts[block.partition].insert(block.number); diff --git a/src/Storages/tests/transform_part_zk_nodes.cpp b/src/Storages/tests/transform_part_zk_nodes.cpp index 2b4f6b7ff5c..35e44bb9446 100644 --- a/src/Storages/tests/transform_part_zk_nodes.cpp +++ b/src/Storages/tests/transform_part_zk_nodes.cpp @@ -76,7 +76,7 @@ try } catch (const Coordination::Exception & e) { - if (e.code == Coordination::ZNONODE) + if (e.code == Coordination::Error::ZNONODE) continue; throw; } diff --git a/utils/zookeeper-dump-tree/main.cpp b/utils/zookeeper-dump-tree/main.cpp index d848ffdad3c..877a4a996b7 100644 --- a/utils/zookeeper-dump-tree/main.cpp +++ b/utils/zookeeper-dump-tree/main.cpp @@ -45,7 +45,7 @@ try } catch (const Coordination::Exception & e) { - if (e.code == Coordination::ZNONODE) + if (e.code == Coordination::Error::ZNONODE) continue; throw; } From e5897bbfb6073b087488ce58e940683ad9f55133 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Fri, 12 Jun 2020 18:59:11 +0300 Subject: [PATCH 111/329] size_t -> ssize_t --- src/Processors/Transforms/AggregatingInOrderTransform.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index 3cac1c9602c..7edeff65ec8 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -100,9 +100,9 @@ void AggregatingInOrderTransform::consume(Chunk chunk) params->aggregator.createStatesAndFillKeyColumnsWithSingleKey(variants, key_columns, key_begin, res_key_columns); ++cur_block_size; } - size_t mid = 0; - size_t high = 0; - size_t low = -1; + ssize_t mid = 0; + ssize_t high = 0; + ssize_t low = -1; /// Will split block into segments with the same key while (key_end != rows) { From 8d9b770da4a74a16c405f90df669bf1d34135fc5 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sat, 13 Jun 2020 00:15:02 +0800 Subject: [PATCH 112/329] ISSUES-7572 fix defaults config level & add replicas_status and prometheus handler --- src/Server/HTTPHandlerFactory.cpp | 93 +++++++++++-------- src/Server/HTTPHandlerFactory.h | 14 +-- src/Server/PrometheusRequestHandler.cpp | 15 +-- src/Server/ReplicasStatusHandler.cpp | 6 +- src/Server/StaticRequestHandler.cpp | 16 ---- .../test_http_handlers_config/test.py | 26 ++++-- .../test_custom_defaults_handlers/config.xml | 10 -- .../test_prometheus_handler/config.xml | 17 ++++ .../test_replicas_status_handler/config.xml | 12 +++ 9 files changed, 112 insertions(+), 97 deletions(-) delete mode 100644 tests/integration/test_http_handlers_config/test_custom_defaults_handlers/config.xml create mode 100644 tests/integration/test_http_handlers_config/test_prometheus_handler/config.xml create mode 100644 tests/integration/test_http_handlers_config/test_replicas_status_handler/config.xml diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp index ec75656a9a8..6459b0aab3b 100644 --- a/src/Server/HTTPHandlerFactory.cpp +++ b/src/Server/HTTPHandlerFactory.cpp @@ -74,55 +74,51 @@ static inline auto createHandlersFactoryFromConfig( for (const auto & key : keys) { - if (!startsWith(key, "rule")) - throw Exception("Unknown element in config: " + prefix + "." + key + ", must be 'rule'", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + if (key == "defaults") + addDefaultHandlersFactory(*main_handler_factory, server, &async_metrics); + else if (startsWith(key, "rule")) + { + const auto & handler_type = server.config().getString(prefix + "." + key + ".handler.type", ""); - const auto & handler_type = server.config().getString(prefix + "." + key + ".handler.type", ""); + if (handler_type.empty()) + throw Exception("Handler type in config is not specified here: " + prefix + "." + key + ".handler.type", + ErrorCodes::INVALID_CONFIG_PARAMETER); - if (handler_type == "root") - addRootHandlerFactory(*main_handler_factory, server); - else if (handler_type == "ping") - addPingHandlerFactory(*main_handler_factory, server); - else if (handler_type == "defaults") - addDefaultHandlersFactory(*main_handler_factory, server, async_metrics); - else if (handler_type == "prometheus") - addPrometheusHandlerFactory(*main_handler_factory, server, async_metrics); - else if (handler_type == "replicas_status") - addReplicasStatusHandlerFactory(*main_handler_factory, server); - else if (handler_type == "static") - main_handler_factory->addHandler(createStaticHandlerFactory(server, prefix + "." + key)); - else if (handler_type == "dynamic_query_handler") - main_handler_factory->addHandler(createDynamicHandlerFactory(server, prefix + "." + key)); - else if (handler_type == "predefined_query_handler") - main_handler_factory->addHandler(createPredefinedHandlerFactory(server, prefix + "." + key)); - else if (handler_type.empty()) - throw Exception("Handler type in config is not specified here: " + - prefix + "." + key + ".handler.type", ErrorCodes::INVALID_CONFIG_PARAMETER); + if (handler_type == "static") + main_handler_factory->addHandler(createStaticHandlerFactory(server, prefix + "." + key)); + else if (handler_type == "dynamic_query_handler") + main_handler_factory->addHandler(createDynamicHandlerFactory(server, prefix + "." + key)); + else if (handler_type == "predefined_query_handler") + main_handler_factory->addHandler(createPredefinedHandlerFactory(server, prefix + "." + key)); + else if (handler_type == "prometheus") + main_handler_factory->addHandler(createPrometheusHandlerFactory(server, async_metrics, prefix + "." + key)); + else if (handler_type == "replicas_status") + main_handler_factory->addHandler(createReplicasStatusHandlerFactory(server, prefix + "." + key)); + else + throw Exception("Unknown handler type '" + handler_type + "' in config here: " + prefix + "." + key + ".handler.type", + ErrorCodes::INVALID_CONFIG_PARAMETER); + } else - throw Exception("Unknown handler type '" + handler_type +"' in config here: " + - prefix + "." + key + ".handler.type",ErrorCodes::INVALID_CONFIG_PARAMETER); + throw Exception("Unknown element in config: " + prefix + "." + key + ", must be 'rule' or 'defaults'", + ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); } return main_handler_factory.release(); } -static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory( - IServer & server, const std::string & name, AsynchronousMetrics & async_metrics) +static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory(IServer & server, const std::string & name, AsynchronousMetrics & async_metrics) { if (server.config().has("http_handlers")) return createHandlersFactoryFromConfig(server, name, "http_handlers", async_metrics); else { auto factory = std::make_unique(name); - - addRootHandlerFactory(*factory, server); - addPingHandlerFactory(*factory, server); - addReplicasStatusHandlerFactory(*factory, server); - addPrometheusHandlerFactory(*factory, server, async_metrics); + addDefaultHandlersFactory(*factory, server, &async_metrics); auto query_handler = std::make_unique>(server, "query"); query_handler->allowPostAndGetParamsRequest(); factory->addHandler(query_handler.release()); + return factory.release(); } } @@ -130,10 +126,7 @@ static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory( static inline Poco::Net::HTTPRequestHandlerFactory * createInterserverHTTPHandlerFactory(IServer & server, const std::string & name) { auto factory = std::make_unique(name); - - addRootHandlerFactory(*factory, server); - addPingHandlerFactory(*factory, server); - addReplicasStatusHandlerFactory(*factory, server); + addDefaultHandlersFactory(*factory, server, nullptr); auto main_handler = std::make_unique>(server); main_handler->allowPostAndGetParamsRequest(); @@ -161,12 +154,32 @@ Poco::Net::HTTPRequestHandlerFactory * createHandlerFactory(IServer & server, As throw Exception("LOGICAL ERROR: Unknown HTTP handler factory name.", ErrorCodes::LOGICAL_ERROR); } -void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics & async_metrics) +static const auto ping_response_expression = "Ok.\n"; +static const auto root_response_expression = "config://http_server_default_response"; + +void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics * async_metrics) { - addRootHandlerFactory(factory, server); - addPingHandlerFactory(factory, server); - addReplicasStatusHandlerFactory(factory, server); - addPrometheusHandlerFactory(factory, server, async_metrics); + auto root_handler = std::make_unique>(server, root_response_expression); + root_handler->attachStrictPath("/")->allowGetAndHeadRequest(); + factory.addHandler(root_handler.release()); + + auto ping_handler = std::make_unique>(server, ping_response_expression); + ping_handler->attachStrictPath("/ping")->allowGetAndHeadRequest(); + factory.addHandler(ping_handler.release()); + + auto replicas_status_handler = std::make_unique>(server); + replicas_status_handler->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest(); + factory.addHandler(replicas_status_handler.release()); + + /// We check that prometheus handler will be served on current (default) port. + /// Otherwise it will be created separately, see below. + if (async_metrics && server.config().has("prometheus") && server.config().getInt("prometheus.port", 0) == 0) + { + auto prometheus_handler = std::make_unique>( + server, PrometheusMetricsWriter(server.config(), "prometheus", *async_metrics)); + prometheus_handler->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics"))->allowGetAndHeadRequest(); + factory.addHandler(prometheus_handler.release()); + } } } diff --git a/src/Server/HTTPHandlerFactory.h b/src/Server/HTTPHandlerFactory.h index ac3a7451338..8e21a13ba18 100644 --- a/src/Server/HTTPHandlerFactory.h +++ b/src/Server/HTTPHandlerFactory.h @@ -103,15 +103,7 @@ private: std::function creator; }; -void addRootHandlerFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server); - -void addPingHandlerFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server); - -void addReplicasStatusHandlerFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server); - -void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics & async_metrics); - -void addPrometheusHandlerFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics & async_metrics); +void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics * async_metrics); Poco::Net::HTTPRequestHandlerFactory * createStaticHandlerFactory(IServer & server, const std::string & config_prefix); @@ -119,6 +111,10 @@ Poco::Net::HTTPRequestHandlerFactory * createDynamicHandlerFactory(IServer & ser Poco::Net::HTTPRequestHandlerFactory * createPredefinedHandlerFactory(IServer & server, const std::string & config_prefix); +Poco::Net::HTTPRequestHandlerFactory * createReplicasStatusHandlerFactory(IServer & server, const std::string & config_prefix); + +Poco::Net::HTTPRequestHandlerFactory * createPrometheusHandlerFactory(IServer & server, AsynchronousMetrics & async_metrics, const std::string & config_prefix); + Poco::Net::HTTPRequestHandlerFactory * createHandlerFactory(IServer & server, AsynchronousMetrics & async_metrics, const std::string & name); } diff --git a/src/Server/PrometheusRequestHandler.cpp b/src/Server/PrometheusRequestHandler.cpp index 0f5df54b002..60deec9b289 100644 --- a/src/Server/PrometheusRequestHandler.cpp +++ b/src/Server/PrometheusRequestHandler.cpp @@ -12,7 +12,7 @@ #include #include -#include +#include namespace DB @@ -41,17 +41,10 @@ void PrometheusRequestHandler::handleRequest( } } -void addPrometheusHandlerFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics & async_metrics) +Poco::Net::HTTPRequestHandlerFactory * createPrometheusHandlerFactory(IServer & server, AsynchronousMetrics & async_metrics, const std::string & config_prefix) { - /// We check that prometheus handler will be served on current (default) port. - /// Otherwise it will be created separately, see below. - if (server.config().has("prometheus") && server.config().getInt("prometheus.port", 0) == 0) - { - auto prometheus_handler = std::make_unique>( - server, PrometheusMetricsWriter(server.config(), "prometheus", async_metrics)); - prometheus_handler->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics"))->allowGetAndHeadRequest(); - factory.addHandler(prometheus_handler.release()); - } + return addFiltersFromConfig(new HandlingRuleHTTPHandlerFactory( + server, PrometheusMetricsWriter(server.config(), config_prefix + ".handler", async_metrics)), server.config(), config_prefix); } } diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index d6bbfdbd090..3606da23ab5 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -106,11 +106,9 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request } } -void addReplicasStatusHandlerFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server) +Poco::Net::HTTPRequestHandlerFactory * createReplicasStatusHandlerFactory(IServer & server, const std::string & config_prefix) { - auto replicas_status_handler = std::make_unique>(server); - replicas_status_handler->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest(); - factory.addHandler(replicas_status_handler.release()); + return addFiltersFromConfig(new HandlingRuleHTTPHandlerFactory(server), server.config(), config_prefix); } } diff --git a/src/Server/StaticRequestHandler.cpp b/src/Server/StaticRequestHandler.cpp index 255e3cab5af..22f32e6a0e7 100644 --- a/src/Server/StaticRequestHandler.cpp +++ b/src/Server/StaticRequestHandler.cpp @@ -155,22 +155,6 @@ StaticRequestHandler::StaticRequestHandler(IServer & server_, const String & exp { } -void addRootHandlerFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server) -{ - static const auto root_response_expression = "config://http_server_default_response"; - - auto root_handler = std::make_unique>(server, root_response_expression); - root_handler->attachStrictPath("/")->allowGetAndHeadRequest(); - factory.addHandler(root_handler.release()); -} - -void addPingHandlerFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server) -{ - auto ping_handler = std::make_unique>(server, "Ok.\n"); - ping_handler->attachStrictPath("/ping")->allowGetAndHeadRequest(); - factory.addHandler(ping_handler.release()); -} - Poco::Net::HTTPRequestHandlerFactory * createStaticHandlerFactory(IServer & server, const std::string & config_prefix) { int status = server.config().getInt(config_prefix + ".handler.status", 200); diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index a38bd3ff343..c18c22acbb2 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -124,12 +124,24 @@ def test_defaults_http_handlers(): assert 200 == cluster.instance.http_request('replicas_status', method='GET').status_code assert 'Ok\n' == cluster.instance.http_request('replicas_status', method='GET').content -def test_custom_defaults_http_handlers(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "custom_defaults_handlers", "test_custom_defaults_handlers")) as cluster: - assert 200 == cluster.instance.http_request('', method='GET').status_code - assert 'Default server response' == cluster.instance.http_request('', method='GET').content +def test_prometheus_handler(): + with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster: + assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code - assert 200 == cluster.instance.http_request('ping', method='GET').status_code - assert 'Ok\n' == cluster.instance.http_request('ping', method='GET').content + assert 404 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'bad'}).status_code - assert 404 == cluster.instance.http_request('replicas_status', method='GET').status_code + assert 404 == cluster.instance.http_request('test_prometheus', method='POST', headers={'XXX': 'xxx'}).status_code + + assert 200 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).status_code + assert 'ClickHouseProfileEvents_Query' in cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).content + +def test_replicas_status_handler(): + with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "replicas_status_handler", "test_replicas_status_handler")) as cluster: + assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code + + assert 404 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'bad'}).status_code + + assert 404 == cluster.instance.http_request('test_replicas_status', method='POST', headers={'XXX': 'xxx'}).status_code + + assert 200 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).status_code + assert 'Ok\n' == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).content diff --git a/tests/integration/test_http_handlers_config/test_custom_defaults_handlers/config.xml b/tests/integration/test_http_handlers_config/test_custom_defaults_handlers/config.xml deleted file mode 100644 index 54008c2c4b8..00000000000 --- a/tests/integration/test_http_handlers_config/test_custom_defaults_handlers/config.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - Default server response - - - - - - diff --git a/tests/integration/test_http_handlers_config/test_prometheus_handler/config.xml b/tests/integration/test_http_handlers_config/test_prometheus_handler/config.xml new file mode 100644 index 00000000000..7c80649cee2 --- /dev/null +++ b/tests/integration/test_http_handlers_config/test_prometheus_handler/config.xml @@ -0,0 +1,17 @@ + + + + + + GET + xxx + /test_prometheus + + replicas_status + true + true + true + + + + diff --git a/tests/integration/test_http_handlers_config/test_replicas_status_handler/config.xml b/tests/integration/test_http_handlers_config/test_replicas_status_handler/config.xml new file mode 100644 index 00000000000..21f7d3b0fc8 --- /dev/null +++ b/tests/integration/test_http_handlers_config/test_replicas_status_handler/config.xml @@ -0,0 +1,12 @@ + + + + + + GET + xxx + /test_replicas_status + replicas_status + + + From f608044690fa4f66d6a7b3aefc3889734bec5c0c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 21:19:16 +0300 Subject: [PATCH 113/329] Fix error --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 8564b996bda..3fbdf2a02f5 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -338,7 +338,7 @@ static void read(int32_t & x, ReadBuffer & in) static void read(Error & x, ReadBuffer & in) { int32_t code; - readBinary(code, in); + read(code, in); x = Error(code); } From 7f52b615e061d6cbd3c493bc599913541875e397 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 12 Jun 2020 21:17:06 +0300 Subject: [PATCH 114/329] Fix bloom filters for String (data skipping indices) bloom filter was broken for the first element, if all of the following conditions satisfied: - they are created on INSERT (in thie case bloom filter hashing uses offsets, in case of OPTIMIZE it does not, since it already has granulars). - the record is not the first in the block - the record is the first per index_granularity (do not confuse this with data skipping index GRANULARITY). - type of the field for indexing is "String" (not FixedString) Because in this case there was incorrect length and *data* for that string. --- src/Interpreters/BloomFilterHash.h | 7 +++---- .../0_stateless/01307_data_skip_bloom_filter.reference | 4 ++++ .../queries/0_stateless/01307_data_skip_bloom_filter.sql | 8 ++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/01307_data_skip_bloom_filter.reference create mode 100644 tests/queries/0_stateless/01307_data_skip_bloom_filter.sql diff --git a/src/Interpreters/BloomFilterHash.h b/src/Interpreters/BloomFilterHash.h index e7411433781..43f5d7b5e87 100644 --- a/src/Interpreters/BloomFilterHash.h +++ b/src/Interpreters/BloomFilterHash.h @@ -196,18 +196,17 @@ struct BloomFilterHash const ColumnString::Chars & data = index_column->getChars(); const ColumnString::Offsets & offsets = index_column->getOffsets(); - ColumnString::Offset current_offset = pos; for (size_t index = 0, size = vec.size(); index < size; ++index) { + ColumnString::Offset current_offset = offsets[index + pos - 1]; + size_t length = offsets[index + pos] - current_offset - 1 /* terminating zero */; UInt64 city_hash = CityHash_v1_0_2::CityHash64( - reinterpret_cast(&data[current_offset]), offsets[index + pos] - current_offset - 1); + reinterpret_cast(&data[current_offset]), length); if constexpr (is_first) vec[index] = city_hash; else vec[index] = CityHash_v1_0_2::Hash128to64(CityHash_v1_0_2::uint128(vec[index], city_hash)); - - current_offset = offsets[index + pos]; } } else if (const auto * fixed_string_index_column = typeid_cast(column)) diff --git a/tests/queries/0_stateless/01307_data_skip_bloom_filter.reference b/tests/queries/0_stateless/01307_data_skip_bloom_filter.reference new file mode 100644 index 00000000000..98fb6a68656 --- /dev/null +++ b/tests/queries/0_stateless/01307_data_skip_bloom_filter.reference @@ -0,0 +1,4 @@ +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01307_data_skip_bloom_filter.sql b/tests/queries/0_stateless/01307_data_skip_bloom_filter.sql new file mode 100644 index 00000000000..832f7140af2 --- /dev/null +++ b/tests/queries/0_stateless/01307_data_skip_bloom_filter.sql @@ -0,0 +1,8 @@ +DROP TABLE IF EXISTS test_01307; +CREATE TABLE test_01307 (id UInt64, val String, INDEX ind val TYPE bloom_filter() GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 2; +INSERT INTO test_01307 (id, val) select number as id, toString(number) as val from numbers(4); +SELECT count() FROM test_01307 WHERE identity(val) = '2'; +SELECT count() FROM test_01307 WHERE val = '2'; +OPTIMIZE TABLE test_01307 FINAL; +SELECT count() FROM test_01307 WHERE identity(val) = '2'; +SELECT count() FROM test_01307 WHERE val = '2'; From 8985d64237e9b7561c778570683f372c4debc4e5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 22:39:48 +0300 Subject: [PATCH 115/329] Fix "Arcadia" build --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 3fbdf2a02f5..9d42e2f7a48 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -52,13 +52,6 @@ namespace CurrentMetrics extern const Metric ZooKeeperWatch; } -namespace DB -{ - namespace ErrorCodes - { - extern const int SUPPORT_IS_DISABLED; - } -} /** ZooKeeper wire protocol. @@ -908,7 +901,8 @@ void ZooKeeper::connect( #if USE_SSL socket = Poco::Net::SecureStreamSocket(); #else - throw Exception{"Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED}; + throw Poco::Exception{ + "Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support."}; #endif } else From a6908515864f1adab9b09c323a24f91e5dd35c67 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jun 2020 00:20:59 +0300 Subject: [PATCH 116/329] Fix style --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 9d42e2f7a48..61a64f2c51f 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -901,8 +901,8 @@ void ZooKeeper::connect( #if USE_SSL socket = Poco::Net::SecureStreamSocket(); #else - throw Poco::Exception{ - "Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support."}; + throw Poco::Exception( + "Communication with ZooKeeper over SSL is disabled because poco library was built without NetSSL support."); #endif } else From 0b1ff4f9cce72ab24e6b9bb909874f81789423ba Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Jun 2020 00:48:10 +0300 Subject: [PATCH 117/329] Update max_rows_to_read in 00945_bloom_filter_index test The expected values was incorrect, since for strings we have 1 and 10 and there will be at least two index granulas, hence 12 rows. --- tests/queries/0_stateless/00945_bloom_filter_index.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.sql b/tests/queries/0_stateless/00945_bloom_filter_index.sql index 6f93ae89a42..d509b99229a 100755 --- a/tests/queries/0_stateless/00945_bloom_filter_index.sql +++ b/tests/queries/0_stateless/00945_bloom_filter_index.sql @@ -43,7 +43,7 @@ SELECT COUNT() FROM bloom_filter_types_test WHERE f32 = 1 SETTINGS max_rows_to_r SELECT COUNT() FROM bloom_filter_types_test WHERE f64 = 1 SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_types_test WHERE date = '1970-01-02' SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_types_test WHERE date_time = toDateTime('1970-01-01 03:00:01', 'Europe/Moscow') SETTINGS max_rows_to_read = 6; -SELECT COUNT() FROM bloom_filter_types_test WHERE str = '1' SETTINGS max_rows_to_read = 6; +SELECT COUNT() FROM bloom_filter_types_test WHERE str = '1' SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_types_test WHERE fixed_string = toFixedString('1', 5) SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_types_test WHERE str IN ( SELECT str FROM bloom_filter_types_test); @@ -122,7 +122,7 @@ SELECT COUNT() FROM bloom_filter_null_types_test WHERE f32 = 1 SETTINGS max_rows SELECT COUNT() FROM bloom_filter_null_types_test WHERE f64 = 1 SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_null_types_test WHERE date = '1970-01-02' SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_null_types_test WHERE date_time = toDateTime('1970-01-01 03:00:01', 'Europe/Moscow') SETTINGS max_rows_to_read = 6; -SELECT COUNT() FROM bloom_filter_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 6; +SELECT COUNT() FROM bloom_filter_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_null_types_test WHERE fixed_string = toFixedString('1', 5) SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_null_types_test WHERE isNull(i8); @@ -150,7 +150,7 @@ CREATE TABLE bloom_filter_lc_null_types_test (order_key UInt64, str LowCardinali INSERT INTO bloom_filter_lc_null_types_test SELECT number AS order_key, toString(number) AS str, toFixedString(toString(number), 5) AS fixed_string FROM system.numbers LIMIT 100; INSERT INTO bloom_filter_lc_null_types_test SELECT 0 AS order_key, NULL AS str, NULL AS fixed_string; -SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 6; +SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE fixed_string = toFixedString('1', 5) SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_lc_null_types_test WHERE isNull(str); From 6869c122a811fed91c9fc953df4d4af89a3a30f7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 21:24:32 +0300 Subject: [PATCH 118/329] Step 1 towards removing leader election: check and modify version of the "log" node --- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 10 +- .../MergeTree/ReplicatedMergeTreeQueue.h | 8 +- src/Storages/StorageReplicatedMergeTree.cpp | 249 ++++++++++++------ src/Storages/StorageReplicatedMergeTree.h | 13 +- 4 files changed, 199 insertions(+), 81 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 09e9cd494ca..8a9dbceba04 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -419,7 +419,7 @@ bool ReplicatedMergeTreeQueue::removeFromVirtualParts(const MergeTreePartInfo & return virtual_parts.remove(part_info); } -void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback) +int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback) { std::lock_guard lock(pull_logs_to_queue_mutex); if (pull_log_blocker.isCancelled()) @@ -428,6 +428,10 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C String index_str = zookeeper->get(replica_path + "/log_pointer"); UInt64 index; + /// The version of "/log" is modified when new entries to merge/mutate/drop appear. + Coordination::Stat stat; + zookeeper->get(zookeeper_path + "/log", &stat); + Strings log_entries = zookeeper->getChildrenWatch(zookeeper_path + "/log", nullptr, watch_callback); /// We update mutations after we have loaded the list of log entries, but before we insert them @@ -561,6 +565,8 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C if (storage.queue_task_handle) storage.queue_task_handle->signalReadyToRun(); } + + return stat.version; } @@ -1630,7 +1636,7 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( } } - queue_.pullLogsToQueue(zookeeper); + merges_version = queue_.pullLogsToQueue(zookeeper); Coordination::GetResponse quorum_status_response = quorum_status_future.get(); if (quorum_status_response.error == Coordination::Error::ZOK) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 4cbb86adb7b..e093e193381 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -271,8 +271,9 @@ public: * If watch_callback is not empty, will call it when new entries appear in the log. * If there were new entries, notifies storage.queue_task_handle. * Additionally loads mutations (so that the set of mutations is always more recent than the queue). + * Return the version of "logs" node (that is updated for every merge/mutation/... added to the log) */ - void pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback = {}); + int32_t pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback = {}); /// Load new mutation entries. If something new is loaded, schedule storage.merge_selecting_task. /// If watch_callback is not empty, will call it when new mutations appear in ZK. @@ -434,6 +435,9 @@ public: bool isMutationFinished(const ReplicatedMergeTreeMutationEntry & mutation) const; + /// The version of "log" node that is used to check that no new merges have appeared. + int32_t getVersion() const { return merges_version; } + private: const ReplicatedMergeTreeQueue & queue; @@ -445,6 +449,8 @@ private: /// Quorum state taken at some later time than prev_virtual_parts. String inprogress_quorum_part; + + int32_t merges_version = -1; }; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e7d9e214995..9d20aeecac1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2402,7 +2402,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() const bool deduplicate = false; /// TODO: read deduplicate option from table config const bool force_ttl = false; - bool success = false; + CreateMergeEntryResult create_result = CreateMergeEntryResult::Other; try { @@ -2422,7 +2422,11 @@ void StorageReplicatedMergeTree::mergeSelectingTask() size_t merges_and_mutations_sum = merges_and_mutations_queued.first + merges_and_mutations_queued.second; if (merges_and_mutations_sum >= storage_settings_ptr->max_replicated_merges_in_queue) { - LOG_TRACE(log, "Number of queued merges ({}) and part mutations ({}) is greater than max_replicated_merges_in_queue ({}), so won't select new parts to merge or mutate.", merges_and_mutations_queued.first, merges_and_mutations_queued.second, storage_settings_ptr->max_replicated_merges_in_queue); + LOG_TRACE(log, "Number of queued merges ({}) and part mutations ({})" + " is greater than max_replicated_merges_in_queue ({}), so won't select new parts to merge or mutate.", + merges_and_mutations_queued.first, + merges_and_mutations_queued.second, + storage_settings_ptr->max_replicated_merges_in_queue); } else { @@ -2434,10 +2438,10 @@ void StorageReplicatedMergeTree::mergeSelectingTask() if (max_source_parts_size_for_merge > 0 && merger_mutator.selectPartsToMerge(future_merged_part, false, max_source_parts_size_for_merge, merge_pred, nullptr)) { - success = createLogEntryToMergeParts(zookeeper, future_merged_part.parts, - future_merged_part.name, future_merged_part.type, deduplicate, force_ttl); + create_result = createLogEntryToMergeParts(zookeeper, future_merged_part.parts, + future_merged_part.name, future_merged_part.type, deduplicate, force_ttl, nullptr, merge_pred.getVersion()); } - /// If there are many mutations in queue it may happen, that we cannot enqueue enough merges to merge all new parts + /// If there are many mutations in queue, it may happen, that we cannot enqueue enough merges to merge all new parts else if (max_source_part_size_for_mutation > 0 && queue.countMutations() > 0 && merges_and_mutations_queued.second < storage_settings_ptr->max_replicated_mutations_in_queue) { @@ -2452,11 +2456,11 @@ void StorageReplicatedMergeTree::mergeSelectingTask() if (!desired_mutation_version) continue; - if (createLogEntryToMutatePart(*part, desired_mutation_version->first, desired_mutation_version->second)) - { - success = true; + create_result = createLogEntryToMutatePart(*part, + desired_mutation_version->first, desired_mutation_version->second, merge_pred.getVersion()); + + if (create_result == CreateMergeEntryResult::Ok) break; - } } } } @@ -2469,11 +2473,15 @@ void StorageReplicatedMergeTree::mergeSelectingTask() if (!is_leader) return; - if (!success) + if (create_result != CreateMergeEntryResult::Ok + && create_result != CreateMergeEntryResult::LogUpdated) + { merge_selecting_task->scheduleAfter(MERGE_SELECTING_SLEEP_MS); + } else + { merge_selecting_task->schedule(); - + } } @@ -2507,14 +2515,15 @@ void StorageReplicatedMergeTree::mutationsFinalizingTask() } -bool StorageReplicatedMergeTree::createLogEntryToMergeParts( +StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::createLogEntryToMergeParts( zkutil::ZooKeeperPtr & zookeeper, const DataPartsVector & parts, const String & merged_name, const MergeTreeDataPartType & merged_part_type, bool deduplicate, bool force_ttl, - ReplicatedMergeTreeLogEntryData * out_log_entry) + ReplicatedMergeTreeLogEntryData * out_log_entry, + int32_t log_version) { std::vector> exists_futures; exists_futures.reserve(parts.size()); @@ -2539,7 +2548,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( } if (!all_in_zk) - return false; + return CreateMergeEntryResult::MissingPart; ReplicatedMergeTreeLogEntryData entry; entry.type = LogEntry::MERGE_PARTS; @@ -2551,21 +2560,43 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( entry.create_time = time(nullptr); for (const auto & part : parts) - { entry.source_parts.push_back(part->name); - } - String path_created = zookeeper->create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); - entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); + Coordination::Requests ops; + Coordination::Responses responses; + + ops.emplace_back(zkutil::makeCreateRequest( + zookeeper_path + "/log/log-", entry.toString(), + zkutil::CreateMode::PersistentSequential)); + + ops.emplace_back(zkutil::makeCheckRequest( + zookeeper_path + "/log", log_version)); + + Coordination::Error code = zookeeper->tryMulti(ops, responses); + + if (code == Coordination::Error::ZOK) + { + String path_created = dynamic_cast(*responses.front()).path_created; + entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); + } + else if (code == Coordination::Error::ZBADVERSION) + { + return CreateMergeEntryResult::LogUpdated; + } + else + { + zkutil::KeeperMultiException::check(code, ops, responses); + } if (out_log_entry) *out_log_entry = entry; - return true; + return CreateMergeEntryResult::Ok; } -bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const IMergeTreeDataPart & part, Int64 mutation_version, int alter_version) +StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::createLogEntryToMutatePart( + const IMergeTreeDataPart & part, Int64 mutation_version, int32_t alter_version, int32_t log_version) { auto zookeeper = getZooKeeper(); @@ -2578,7 +2609,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const IMergeTreeData enqueuePartForCheck(part.name); } - return false; + return CreateMergeEntryResult::MissingPart; } MergeTreePartInfo new_part_info = part.info; @@ -2594,8 +2625,23 @@ bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const IMergeTreeData entry.create_time = time(nullptr); entry.alter_version = alter_version; - zookeeper->create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); - return true; + Coordination::Requests ops; + Coordination::Responses responses; + + ops.emplace_back(zkutil::makeCreateRequest( + zookeeper_path + "/log/log-", entry.toString(), + zkutil::CreateMode::PersistentSequential)); + + ops.emplace_back(zkutil::makeCheckRequest( + zookeeper_path + "/log", log_version)); + + Coordination::Error code = zookeeper->tryMulti(ops, responses); + + if (code == Coordination::Error::ZBADVERSION) + return CreateMergeEntryResult::LogUpdated; + + zkutil::KeeperMultiException::check(code, ops, responses); + return CreateMergeEntryResult::Ok; } @@ -3377,7 +3423,8 @@ BlockOutputStreamPtr StorageReplicatedMergeTree::write(const ASTPtr & /*query*/, } -bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & query_context) +bool StorageReplicatedMergeTree::optimize( + const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & query_context) { assertNotReadonly(); @@ -3387,14 +3434,11 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p return true; } + constexpr size_t max_retries = 10; + std::vector merge_entries; { - /// We must select parts for merge under merge_selecting_mutex because other threads - /// (merge_selecting_thread or OPTIMIZE queries) could assign new merges. - std::lock_guard merge_selecting_lock(merge_selecting_mutex); - auto zookeeper = getZooKeeper(); - ReplicatedMergeTreeMergePredicate can_merge = queue.getMergePredicate(zookeeper); auto handle_noop = [&] (const String & message) { @@ -3418,52 +3462,94 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p for (const String & partition_id : partition_ids) { - FutureMergedMutatedPart future_merged_part; - bool selected = merger_mutator.selectAllPartsToMergeWithinPartition( - future_merged_part, disk_space, can_merge, partition_id, true, nullptr); - ReplicatedMergeTreeLogEntryData merge_entry; - if (selected && !createLogEntryToMergeParts(zookeeper, future_merged_part.parts, - future_merged_part.name, future_merged_part.type, deduplicate, force_ttl, &merge_entry)) - return handle_noop("Can't create merge queue node in ZooKeeper"); - if (merge_entry.type != ReplicatedMergeTreeLogEntryData::Type::EMPTY) + size_t try_no = 0; + for (; try_no < max_retries; ++try_no) + { + /// We must select parts for merge under merge_selecting_mutex because other threads + /// (merge_selecting_thread or OPTIMIZE queries) could assign new merges. + std::lock_guard merge_selecting_lock(merge_selecting_mutex); + ReplicatedMergeTreeMergePredicate can_merge = queue.getMergePredicate(zookeeper); + + FutureMergedMutatedPart future_merged_part; + bool selected = merger_mutator.selectAllPartsToMergeWithinPartition( + future_merged_part, disk_space, can_merge, partition_id, true, nullptr); + + if (!selected) + break; + + ReplicatedMergeTreeLogEntryData merge_entry; + CreateMergeEntryResult create_result = createLogEntryToMergeParts( + zookeeper, future_merged_part.parts, + future_merged_part.name, future_merged_part.type, deduplicate, force_ttl, + &merge_entry, can_merge.getVersion()); + + if (create_result == CreateMergeEntryResult::MissingPart) + return handle_noop("Can't create merge queue node in ZooKeeper, because some parts are missing"); + + if (create_result == CreateMergeEntryResult::LogUpdated) + continue; + merge_entries.push_back(std::move(merge_entry)); + break; + } + if (try_no == max_retries) + return handle_noop("Can't create merge queue node in ZooKeeper, because log was updated in every of " + + toString(max_retries) + " tries"); } } else { - FutureMergedMutatedPart future_merged_part; - String disable_reason; - bool selected = false; - if (!partition) - { - selected = merger_mutator.selectPartsToMerge( - future_merged_part, true, storage_settings_ptr->max_bytes_to_merge_at_max_space_in_pool, can_merge, &disable_reason); - } - else + size_t try_no = 0; + for (; try_no < max_retries; ++try_no) { + std::lock_guard merge_selecting_lock(merge_selecting_mutex); + ReplicatedMergeTreeMergePredicate can_merge = queue.getMergePredicate(zookeeper); - UInt64 disk_space = getStoragePolicy()->getMaxUnreservedFreeSpace(); - String partition_id = getPartitionIDFromQuery(partition, query_context); - selected = merger_mutator.selectAllPartsToMergeWithinPartition( - future_merged_part, disk_space, can_merge, partition_id, final, &disable_reason); - } + FutureMergedMutatedPart future_merged_part; + String disable_reason; + bool selected = false; + if (!partition) + { + selected = merger_mutator.selectPartsToMerge( + future_merged_part, true, storage_settings_ptr->max_bytes_to_merge_at_max_space_in_pool, can_merge, &disable_reason); + } + else + { - if (!selected) - { - std::stringstream message; - message << "Cannot select parts for optimization"; - if (!disable_reason.empty()) - message << ": " << disable_reason; - LOG_INFO(log, message.str()); - return handle_noop(message.str()); - } + UInt64 disk_space = getStoragePolicy()->getMaxUnreservedFreeSpace(); + String partition_id = getPartitionIDFromQuery(partition, query_context); + selected = merger_mutator.selectAllPartsToMergeWithinPartition( + future_merged_part, disk_space, can_merge, partition_id, final, &disable_reason); + } + + if (!selected) + { + std::stringstream message; + message << "Cannot select parts for optimization"; + if (!disable_reason.empty()) + message << ": " << disable_reason; + LOG_INFO(log, message.str()); + return handle_noop(message.str()); + } + + ReplicatedMergeTreeLogEntryData merge_entry; + CreateMergeEntryResult create_result = createLogEntryToMergeParts( + zookeeper, future_merged_part.parts, + future_merged_part.name, future_merged_part.type, deduplicate, force_ttl, + &merge_entry, can_merge.getVersion()); + + if (create_result == CreateMergeEntryResult::MissingPart) + return handle_noop("Can't create merge queue node in ZooKeeper, because some parts are missing"); + + if (create_result == CreateMergeEntryResult::LogUpdated) + continue; - ReplicatedMergeTreeLogEntryData merge_entry; - if (!createLogEntryToMergeParts(zookeeper, future_merged_part.parts, - future_merged_part.name, future_merged_part.type, deduplicate, force_ttl, &merge_entry)) - return handle_noop("Can't create merge queue node in ZooKeeper"); - if (merge_entry.type != ReplicatedMergeTreeLogEntryData::Type::EMPTY) merge_entries.push_back(std::move(merge_entry)); + break; + } + if (try_no == max_retries) + return handle_noop("Can't create merge queue node in ZooKeeper, because log was updated in every of " + + toString(max_retries) + " tries"); } } @@ -3537,7 +3623,8 @@ void StorageReplicatedMergeTree::alter( return; } - auto ast_to_str = [](ASTPtr query) -> String { + auto ast_to_str = [](ASTPtr query) -> String + { if (!query) return ""; return queryToString(query); @@ -3587,7 +3674,6 @@ void StorageReplicatedMergeTree::alter( String new_columns_str = future_metadata.columns.toString(); ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, -1)); - if (ast_to_str(current_metadata.settings_ast) != ast_to_str(future_metadata.settings_ast)) { lockStructureExclusively( @@ -3609,10 +3695,12 @@ void StorageReplicatedMergeTree::alter( alter_entry->alter_version = new_metadata_version; alter_entry->create_time = time(nullptr); - auto maybe_mutation_commands = params.getMutationCommands(current_metadata, query_context.getSettingsRef().materialize_ttl_after_modify, query_context); + auto maybe_mutation_commands = params.getMutationCommands( + current_metadata, query_context.getSettingsRef().materialize_ttl_after_modify, query_context); alter_entry->have_mutation = !maybe_mutation_commands.empty(); - ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeCreateRequest( + zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); std::optional lock_holder; @@ -4015,7 +4103,8 @@ StorageReplicatedMergeTree::allocateBlockNumber( } -Strings StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry(const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active) +Strings StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry( + const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active) { LOG_DEBUG(log, "Waiting for all replicas to process {}", entry.znode_name); @@ -4040,7 +4129,8 @@ Strings StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry(const Re } -bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & replica, const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active) +bool StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry( + const String & replica, const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active) { String entry_str = entry.toString(); String log_node_name; @@ -5185,6 +5275,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ } } + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); /// Just update version ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); Transaction transaction(*this); @@ -5370,7 +5461,8 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta } } - ops.emplace_back(zkutil::makeCreateRequest(dest_table_storage->zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeCreateRequest(dest_table_storage->zookeeper_path + "/log/log-", + entry.toString(), zkutil::CreateMode::PersistentSequential)); { Transaction transaction(*dest_table_storage); @@ -5417,11 +5509,13 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta Coordination::Requests ops_dest; - ops_dest.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry_delete.toString(), zkutil::CreateMode::PersistentSequential)); + ops_dest.emplace_back(zkutil::makeCreateRequest( + zookeeper_path + "/log/log-", entry_delete.toString(), zkutil::CreateMode::PersistentSequential)); + ops_dest.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); /// Just update version op_results = zookeeper->multi(ops_dest); - log_znode_path = dynamic_cast(*op_results.back()).path_created; + log_znode_path = dynamic_cast(*op_results.front()).path_created; entry_delete.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); if (query_context.getSettingsRef().replication_alter_partitions_sync > 1) @@ -5577,7 +5671,12 @@ bool StorageReplicatedMergeTree::dropPartsInPartition( entry.detach = detach; entry.create_time = time(nullptr); - String log_znode_path = zookeeper.create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); /// Just update version. + Coordination::Responses responses = zookeeper.multi(ops); + + String log_znode_path = dynamic_cast(*responses.front()).path_created; entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); return true; diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 18c691046d6..ec38eb7e842 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -425,16 +425,23 @@ private: * Call when merge_selecting_mutex is locked. * Returns false if any part is not in ZK. */ - bool createLogEntryToMergeParts( + enum class CreateMergeEntryResult { Ok, MissingPart, LogUpdated, Other }; + + CreateMergeEntryResult createLogEntryToMergeParts( zkutil::ZooKeeperPtr & zookeeper, const DataPartsVector & parts, const String & merged_name, const MergeTreeDataPartType & merged_part_type, bool deduplicate, bool force_ttl, - ReplicatedMergeTreeLogEntryData * out_log_entry = nullptr); + ReplicatedMergeTreeLogEntryData * out_log_entry, + int32_t log_version); - bool createLogEntryToMutatePart(const IMergeTreeDataPart & part, Int64 mutation_version, int alter_version); + CreateMergeEntryResult createLogEntryToMutatePart( + const IMergeTreeDataPart & part, + Int64 mutation_version, + int32_t alter_version, + int32_t log_version); /// Exchange parts. From 833c8178ae51718008b76a1ebd90fb6988088b03 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 22:01:44 +0300 Subject: [PATCH 119/329] Whitespaces --- src/Storages/StorageReplicatedMergeTree.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9d20aeecac1..0e2d138334e 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2605,7 +2605,8 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c { if (part.modification_time + MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER < time(nullptr)) { - LOG_WARNING(log, "Part {} (that was selected for mutation) with age {} seconds exists locally but not in ZooKeeper. Won't mutate that part and will check it.", part.name, (time(nullptr) - part.modification_time)); + LOG_WARNING(log, "Part {} (that was selected for mutation) with age {} seconds exists locally but not in ZooKeeper." + " Won't mutate that part and will check it.", part.name, (time(nullptr) - part.modification_time)); enqueuePartForCheck(part.name); } From 902774c89cc12671191f82ee4403fd8d5d913d3c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 22:02:00 +0300 Subject: [PATCH 120/329] Removed default value of constructor --- src/Common/ZooKeeper/LeaderElection.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Common/ZooKeeper/LeaderElection.h b/src/Common/ZooKeeper/LeaderElection.h index dca87efe7c2..f8a4d56dc76 100644 --- a/src/Common/ZooKeeper/LeaderElection.h +++ b/src/Common/ZooKeeper/LeaderElection.h @@ -36,7 +36,12 @@ public: * It means that different participants of leader election have different identifiers * and existence of more than one ephemeral node with same identifier indicates an error. */ - LeaderElection(DB::BackgroundSchedulePool & pool_, const std::string & path_, ZooKeeper & zookeeper_, LeadershipHandler handler_, const std::string & identifier_ = "") + LeaderElection( + DB::BackgroundSchedulePool & pool_, + const std::string & path_, + ZooKeeper & zookeeper_, + LeadershipHandler handler_, + const std::string & identifier_) : pool(pool_), path(path_), zookeeper(zookeeper_), handler(handler_), identifier(identifier_) , log_name("LeaderElection (" + path + ")") , log(&Poco::Logger::get(log_name)) From 52ac009754a40d71a4b0e737518d64ed40bd44ad Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 23:22:55 +0300 Subject: [PATCH 121/329] Update versions on merge and mutation --- .../MergeTree/MergeTreeDataMergerMutator.h | 1 - src/Storages/StorageReplicatedMergeTree.cpp | 16 ++++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index 385ada72fdd..7c2ee53fc1d 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -82,7 +82,6 @@ public: const AllowedMergingPredicate & can_merge, String * out_disable_reason = nullptr); - /** Select all the parts in the specified partition for merge, if possible. * final - choose to merge even a single part - that is, allow to merge one part "with itself". */ diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 0e2d138334e..57535466558 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2569,8 +2569,8 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); - ops.emplace_back(zkutil::makeCheckRequest( - zookeeper_path + "/log", log_version)); + ops.emplace_back(zkutil::makeSetRequest( + zookeeper_path + "/log", "", log_version)); /// Check and update version. Coordination::Error code = zookeeper->tryMulti(ops, responses); @@ -2578,9 +2578,12 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c { String path_created = dynamic_cast(*responses.front()).path_created; entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); + + LOG_TRACE(log, "Created log entry {} for merge {}", path_created, merged_name); } else if (code == Coordination::Error::ZBADVERSION) { + LOG_TRACE(log, "Log entry is not created for merge {} because log was updated", merged_name); return CreateMergeEntryResult::LogUpdated; } else @@ -2633,15 +2636,20 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); - ops.emplace_back(zkutil::makeCheckRequest( - zookeeper_path + "/log", log_version)); + ops.emplace_back(zkutil::makeSetRequest( + zookeeper_path + "/log", "", log_version)); /// Check and update version. Coordination::Error code = zookeeper->tryMulti(ops, responses); if (code == Coordination::Error::ZBADVERSION) + { + LOG_TRACE(log, "Log entry is not created for mutation {} because log was updated", new_part_name); return CreateMergeEntryResult::LogUpdated; + } zkutil::KeeperMultiException::check(code, ops, responses); + + LOG_TRACE(log, "Created log entry for mutation {}", new_part_name); return CreateMergeEntryResult::Ok; } From 22dd47c13fc97b8adf726fc42ca94847f1c76dd6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 23:24:17 +0300 Subject: [PATCH 122/329] Added a test --- .../01307_multiple_leaders.reference | 2 + .../0_stateless/01307_multiple_leaders.sh | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tests/queries/0_stateless/01307_multiple_leaders.reference create mode 100755 tests/queries/0_stateless/01307_multiple_leaders.sh diff --git a/tests/queries/0_stateless/01307_multiple_leaders.reference b/tests/queries/0_stateless/01307_multiple_leaders.reference new file mode 100644 index 00000000000..576441b288d --- /dev/null +++ b/tests/queries/0_stateless/01307_multiple_leaders.reference @@ -0,0 +1,2 @@ +2000 1999000 +2000 1999000 diff --git a/tests/queries/0_stateless/01307_multiple_leaders.sh b/tests/queries/0_stateless/01307_multiple_leaders.sh new file mode 100755 index 00000000000..0bf5e0b13bf --- /dev/null +++ b/tests/queries/0_stateless/01307_multiple_leaders.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT -n --query " +DROP TABLE IF EXISTS r0; +DROP TABLE IF EXISTS r1; + +CREATE TABLE r0 (x UInt64) ENGINE = ReplicatedMergeTree('/test/table', 'r0') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M'; +CREATE TABLE r1 (x UInt64) ENGINE = ReplicatedMergeTree('/test/table', 'r1') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M'; +" + +function thread() +{ + REPLICA=$1 + ITERATIONS=$2 + + $CLICKHOUSE_CLIENT --max_block_size 1 --min_insert_block_size_rows 0 --min_insert_block_size_bytes 0 --query "INSERT INTO r$REPLICA SELECT number * 2 + $REPLICA FROM numbers($ITERATIONS)" +} + + +thread 0 1000 & +thread 1 1000 & + +wait + +$CLICKHOUSE_CLIENT -n --query " +SYSTEM SYNC REPLICA r0; +SYSTEM SYNC REPLICA r1; + +SELECT count(), sum(x) FROM r0; +SELECT count(), sum(x) FROM r1; + +DROP TABLE r0; +DROP TABLE r1; +" From b8b55a5b9911c2e0b551274d507a78cf4dbe739e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 22:04:18 +0300 Subject: [PATCH 123/329] More LeaderElection to Storage/MergeTree --- src/{Common/ZooKeeper => Storages/MergeTree}/LeaderElection.h | 4 ++-- src/Storages/StorageReplicatedMergeTree.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/{Common/ZooKeeper => Storages/MergeTree}/LeaderElection.h (97%) diff --git a/src/Common/ZooKeeper/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h similarity index 97% rename from src/Common/ZooKeeper/LeaderElection.h rename to src/Storages/MergeTree/LeaderElection.h index f8a4d56dc76..c94e3e27e5a 100644 --- a/src/Common/ZooKeeper/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -1,11 +1,11 @@ #pragma once -#include "ZooKeeper.h" -#include "KeeperException.h" #include #include #include #include +#include +#include #include diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index ec38eb7e842..382cf7ac469 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -19,12 +19,12 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include From 85c0706901de09a6a327d3d953a509a6ef2376b3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 22:19:01 +0300 Subject: [PATCH 124/329] Step 2: allow multiple leaders --- src/Storages/MergeTree/LeaderElection.h | 45 ++++++++++++------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/src/Storages/MergeTree/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h index c94e3e27e5a..680ffe4992f 100644 --- a/src/Storages/MergeTree/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -23,7 +23,16 @@ namespace CurrentMetrics namespace zkutil { -/** Implements leader election algorithm described here: http://zookeeper.apache.org/doc/r3.4.5/recipes.html#sc_leaderElection +/** Initially was used to implement leader election algorithm described here: + * http://zookeeper.apache.org/doc/r3.4.5/recipes.html#sc_leaderElection + * + * But then we decided to get rid of leader election, so every replica can become leader. + * For now, every replica can become leader if there is no leader among replicas with old version. + * + * Replicas with old versions participate in leader election with ephemeral sequential nodes. + * If the node is first, then replica is leader. + * Replicas with new versions creates persistent sequential nodes. + * If the first node is persistent, then all replicas with new versions become leaders. */ class LeaderElection { @@ -67,16 +76,13 @@ public: private: DB::BackgroundSchedulePool & pool; DB::BackgroundSchedulePool::TaskHolder task; - std::string path; + const std::string path; ZooKeeper & zookeeper; LeadershipHandler handler; std::string identifier; std::string log_name; Poco::Logger * log; - EphemeralNodeHolderPtr node; - std::string node_name; - std::atomic shutdown_called {false}; CurrentMetrics::Increment metric_increment{CurrentMetrics::LeaderElection}; @@ -84,43 +90,35 @@ private: void createNode() { shutdown_called = false; - node = EphemeralNodeHolder::createSequential(path + "/leader_election-", zookeeper, identifier); - - std::string node_path = node->getPath(); - node_name = node_path.substr(node_path.find_last_of('/') + 1); - + zookeeper.create(path + "/leader_election-", identifier, CreateMode::PersistentSequential); task->activateAndSchedule(); } void releaseNode() { shutdown(); - node = nullptr; } void threadFunction() { - bool success = false; - try { Strings children = zookeeper.getChildren(path); - std::sort(children.begin(), children.end()); - auto it = std::lower_bound(children.begin(), children.end(), node_name); - if (it == children.end() || *it != node_name) + if (children.empty()) throw Poco::Exception("Assertion failed in LeaderElection"); - if (it == children.begin()) + std::sort(children.begin(), children.end()); + + Coordination::Stat stat; + zookeeper.get(path + "/" + children.front(), &stat); + + if (!stat.ephemeralOwner) { + /// It is sequential node - we can become leader. ProfileEvents::increment(ProfileEvents::LeaderElectionAcquiredLeadership); handler(); return; } - - if (!zookeeper.existsWatch(path + "/" + *(it - 1), nullptr, task->getWatchCallback())) - task->schedule(); - - success = true; } catch (const KeeperException & e) { @@ -134,8 +132,7 @@ private: DB::tryLogCurrentException(log); } - if (!success) - task->scheduleAfter(10 * 1000); + task->scheduleAfter(10 * 1000); } }; From ab00e343054eb1dee01a0c0d87da5bc2875dfc28 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 22:38:36 +0300 Subject: [PATCH 125/329] Miscellaneous --- src/Storages/MergeTree/LeaderElection.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h index 680ffe4992f..725ab61e877 100644 --- a/src/Storages/MergeTree/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -70,7 +70,7 @@ public: ~LeaderElection() { - releaseNode(); + shutdown(); } private: @@ -94,11 +94,6 @@ private: task->activateAndSchedule(); } - void releaseNode() - { - shutdown(); - } - void threadFunction() { try From cfef7ba6920560442543ffb40fb5be89693f3a0c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 23:23:15 +0300 Subject: [PATCH 126/329] Whitespace --- src/Interpreters/SystemLog.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index d79edde7052..b432cd8803b 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -95,7 +95,6 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi if (asynchronous_metric_log) logs.emplace_back(asynchronous_metric_log.get()); - try { for (auto & log : logs) From 6ff671b092b65b9b016248ebfcc936d4bd5684d7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 23:32:31 +0300 Subject: [PATCH 127/329] Improvement --- src/Storages/MergeTree/LeaderElection.h | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h index 725ab61e877..36209d6c003 100644 --- a/src/Storages/MergeTree/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -29,8 +29,12 @@ namespace zkutil * But then we decided to get rid of leader election, so every replica can become leader. * For now, every replica can become leader if there is no leader among replicas with old version. * + * It's tempting to remove this class at all, but we have to maintain it, + * to maintain compatibility when replicas with different versions work on the same cluster + * (this is allowed for short time period during cluster update). + * * Replicas with old versions participate in leader election with ephemeral sequential nodes. - * If the node is first, then replica is leader. + * If the node is first, then replica is the leader. * Replicas with new versions creates persistent sequential nodes. * If the first node is persistent, then all replicas with new versions become leaders. */ @@ -90,6 +94,17 @@ private: void createNode() { shutdown_called = false; + + /// If there is at least one persistent node, we don't have to create another. + Strings children = zookeeper.getChildren(path); + for (const auto & child : children) + { + Coordination::Stat stat; + zookeeper.get(path + "/" + child, &stat); + if (!stat.ephemeralOwner) + return; + } + zookeeper.create(path + "/leader_election-", identifier, CreateMode::PersistentSequential); task->activateAndSchedule(); } From 21897f2abd451e5c80a9e9cf14bf7ce6f0d83bfe Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 23:38:43 +0300 Subject: [PATCH 128/329] Instrument --- src/Common/ProfileEvents.cpp | 5 +++++ src/Storages/StorageReplicatedMergeTree.cpp | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 8393ea85112..a0eb7a5fb48 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -203,6 +203,11 @@ \ M(CannotWriteToWriteBufferDiscard, "Number of stack traces dropped by query profiler or signal handler because pipe is full or cannot write to pipe.") \ M(QueryProfilerSignalOverruns, "Number of times we drop processing of a signal due to overrun plus the number of signals that OS has not delivered due to overrun.") \ + \ + M(CreatedLogEntryForMerge, "Successfully created log entry to merge parts in ReplicatedMergeTree.") \ + M(NotCreatedLogEntryForMerge, "Log entry to merge parts in ReplicatedMergeTree is not created due to concurrent log update by another replica.") \ + M(CreatedLogEntryForMutation, "Successfully created log entry to mutate parts in ReplicatedMergeTree.") \ + M(NotCreatedLogEntryForMutation, "Log entry to mutate parts in ReplicatedMergeTree is not created due to concurrent log update by another replica.") \ namespace ProfileEvents { diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 57535466558..eb395ff55c0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -71,6 +71,10 @@ namespace ProfileEvents extern const Event ReplicatedPartFetches; extern const Event DataAfterMergeDiffersFromReplica; extern const Event DataAfterMutationDiffersFromReplica; + extern const Event CreatedLogEntryForMerge; + extern const Event NotCreatedLogEntryForMerge; + extern const Event CreatedLogEntryForMutation; + extern const Event NotCreatedLogEntryForMutation; } namespace CurrentMetrics @@ -2579,10 +2583,12 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c String path_created = dynamic_cast(*responses.front()).path_created; entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); + ProfileEvents::increment(ProfileEvents::CreatedLogEntryForMerge); LOG_TRACE(log, "Created log entry {} for merge {}", path_created, merged_name); } else if (code == Coordination::Error::ZBADVERSION) { + ProfileEvents::increment(ProfileEvents::NotCreatedLogEntryForMerge); LOG_TRACE(log, "Log entry is not created for merge {} because log was updated", merged_name); return CreateMergeEntryResult::LogUpdated; } @@ -2643,12 +2649,14 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c if (code == Coordination::Error::ZBADVERSION) { + ProfileEvents::increment(ProfileEvents::NotCreatedLogEntryForMutation); LOG_TRACE(log, "Log entry is not created for mutation {} because log was updated", new_part_name); return CreateMergeEntryResult::LogUpdated; } zkutil::KeeperMultiException::check(code, ops, responses); + ProfileEvents::increment(ProfileEvents::CreatedLogEntryForMutation); LOG_TRACE(log, "Created log entry for mutation {}", new_part_name); return CreateMergeEntryResult::Ok; } From 6f0db5ef108b1f13351e4792d427043417187728 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2020 23:42:31 +0300 Subject: [PATCH 129/329] Fix error --- src/Storages/MergeTree/LeaderElection.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h index 36209d6c003..ef6b68bbe15 100644 --- a/src/Storages/MergeTree/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -102,7 +102,11 @@ private: Coordination::Stat stat; zookeeper.get(path + "/" + child, &stat); if (!stat.ephemeralOwner) + { + ProfileEvents::increment(ProfileEvents::LeaderElectionAcquiredLeadership); + handler(); return; + } } zookeeper.create(path + "/leader_election-", identifier, CreateMode::PersistentSequential); @@ -124,7 +128,7 @@ private: if (!stat.ephemeralOwner) { - /// It is sequential node - we can become leader. + /// It is persistent node - we can become leader. ProfileEvents::increment(ProfileEvents::LeaderElectionAcquiredLeadership); handler(); return; From 18f8861fa0df4ac2d1d57c5fc16dca651f4081df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jun 2020 00:19:08 +0300 Subject: [PATCH 130/329] Better test --- .../0_stateless/01307_multiple_leaders.sh | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/tests/queries/0_stateless/01307_multiple_leaders.sh b/tests/queries/0_stateless/01307_multiple_leaders.sh index 0bf5e0b13bf..b16feaeb591 100755 --- a/tests/queries/0_stateless/01307_multiple_leaders.sh +++ b/tests/queries/0_stateless/01307_multiple_leaders.sh @@ -5,35 +5,28 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -n --query " -DROP TABLE IF EXISTS r0; -DROP TABLE IF EXISTS r1; +NUM_REPLICAS=2 +DATA_SIZE=1000 -CREATE TABLE r0 (x UInt64) ENGINE = ReplicatedMergeTree('/test/table', 'r0') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M'; -CREATE TABLE r1 (x UInt64) ENGINE = ReplicatedMergeTree('/test/table', 'r1') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M'; -" +SEQ=$(seq 0 $(($NUM_REPLICAS - 1))) + +for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "DROP TABLE IF EXISTS r$REPLICA"; done +for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE r$REPLICA (x UInt64) ENGINE = ReplicatedMergeTree('/test/table', 'r$REPLICA') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M';"; done function thread() { REPLICA=$1 ITERATIONS=$2 - $CLICKHOUSE_CLIENT --max_block_size 1 --min_insert_block_size_rows 0 --min_insert_block_size_bytes 0 --query "INSERT INTO r$REPLICA SELECT number * 2 + $REPLICA FROM numbers($ITERATIONS)" + $CLICKHOUSE_CLIENT --max_block_size 1 --min_insert_block_size_rows 0 --min_insert_block_size_bytes 0 --query "INSERT INTO r$REPLICA SELECT number * $NUM_REPLICAS + $REPLICA FROM numbers($ITERATIONS)" } - -thread 0 1000 & -thread 1 1000 & +for REPLICA in $SEQ; do + thread $REPLICA $DATA_SIZE & +done wait -$CLICKHOUSE_CLIENT -n --query " -SYSTEM SYNC REPLICA r0; -SYSTEM SYNC REPLICA r1; - -SELECT count(), sum(x) FROM r0; -SELECT count(), sum(x) FROM r1; - -DROP TABLE r0; -DROP TABLE r1; -" +for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "SYSTEM SYNC REPLICA r$REPLICA"; done +for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "SELECT count(), sum(x) FROM r$REPLICA"; done +for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "DROP TABLE r$REPLICA"; done From de96296e019887f925c998bef97e42bafaa964e8 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sat, 13 Jun 2020 10:17:02 +0800 Subject: [PATCH 131/329] ISSUES-7572 fix build failure --- src/Server/ReplicasStatusHandler.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 3606da23ab5..5ead756ee1e 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace DB From 558912be7d4e4c8da7b366be3da00e5d2c2c1905 Mon Sep 17 00:00:00 2001 From: Tom Bombadil <565258751@qq.com> Date: Sat, 13 Jun 2020 10:47:23 +0800 Subject: [PATCH 132/329] Update lazy.md (#11620) * Update lazy.md Optimize the doc translation for Simplified-Chinese version. * Update lazy.md Co-authored-by: Ivan Blinkov --- docs/zh/engines/database-engines/lazy.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/zh/engines/database-engines/lazy.md b/docs/zh/engines/database-engines/lazy.md index c0a08e37559..700eb4b3b25 100644 --- a/docs/zh/engines/database-engines/lazy.md +++ b/docs/zh/engines/database-engines/lazy.md @@ -1,15 +1,13 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 toc_title: "\u61D2\u60F0" --- -# 懒惰 {#lazy} +# 延时引擎Lazy {#lazy} -仅将表保留在RAM中 `expiration_time_in_seconds` 上次访问后几秒钟。 只能与\*日志表一起使用。 +在距最近一次访问间隔`expiration_time_in_seconds`时间段内，将表保存在内存中，仅适用于 \*Log引擎表 -它针对存储许多小\*日志表进行了优化，访问之间存在较长的时间间隔。 +由于针对这类表的访问间隔较长，对保存大量小的 \*Log引擎表进行了优化， ## 创建数据库 {#creating-a-database} From 09a37db2a3e9c12a47f44dc684173a62e06aafb9 Mon Sep 17 00:00:00 2001 From: Tom Bombadil <565258751@qq.com> Date: Sat, 13 Jun 2020 10:48:50 +0800 Subject: [PATCH 133/329] Update clickhouse-benchmark.md (#11619) * Update clickhouse-benchmark.md Optimize doc translation for Simplized-Chinese version,author by TomBombadil * Update clickhouse-benchmark.md Co-authored-by: Ivan Blinkov --- .../utilities/clickhouse-benchmark.md | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/docs/zh/operations/utilities/clickhouse-benchmark.md b/docs/zh/operations/utilities/clickhouse-benchmark.md index d1e83cb9789..1c255f621c0 100644 --- a/docs/zh/operations/utilities/clickhouse-benchmark.md +++ b/docs/zh/operations/utilities/clickhouse-benchmark.md @@ -1,11 +1,9 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 61 -toc_title: "\uFF82\u6697\uFF6A\uFF82\u6C3E\u73AF\u50AC\uFF82\u56E3" +toc_title: "性能测试" --- -# ﾂ暗ｪﾂ氾环催ﾂ団 {#clickhouse-benchmark} +# 性能测试 {#clickhouse-benchmark} 连接到ClickHouse服务器并重复发送指定的查询。 @@ -21,7 +19,7 @@ $ echo "single query" | clickhouse-benchmark [keys] $ clickhouse-benchmark [keys] <<< "single query" ``` -如果要发送一组查询，请创建一个文本文件，并将每个查询放在此文件中的单个字符串上。 例如: +如果要发送一组查询，请创建一个文本文件，并将每个查询的字符串放在此文件中。 例如: ``` sql SELECT * FROM system.numbers LIMIT 10000000 @@ -34,15 +32,15 @@ SELECT 1 clickhouse-benchmark [keys] < queries_file ``` -## 键 {#clickhouse-benchmark-keys} +## keys参数 {#clickhouse-benchmark-keys} - `-c N`, `--concurrency=N` — Number of queries that `clickhouse-benchmark` 同时发送。 默认值：1。 - `-d N`, `--delay=N` — Interval in seconds between intermediate reports (set 0 to disable reports). Default value: 1. -- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. 为 [比较模式](#clickhouse-benchmark-comparison-mode) 您可以使用多个 `-h` 钥匙 +- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. 为 [比较模式](#clickhouse-benchmark-comparison-mode) 您可以使用多个 `-h` 参数 - `-p N`, `--port=N` — Server port. Default value: 9000. For the [比较模式](#clickhouse-benchmark-comparison-mode) 您可以使用多个 `-p` 钥匙 -- `-i N`, `--iterations=N` — Total number of queries. Default value: 0. -- `-r`, `--randomize` — Random order of queries execution if there is more then one input query. -- `-s`, `--secure` — Using TLS connection. +- `-i N`, `--iterations=N` — 查询的总次数. Default value: 0. +- `-r`, `--randomize` — 有多个查询时，以随机顺序执行. +- `-s`, `--secure` — 使用TLS安全连接. - `-t N`, `--timelimit=N` — Time limit in seconds. `clickhouse-benchmark` 达到指定的时间限制时停止发送查询。 默认值：0（禁用时间限制）。 - `--confidence=N` — Level of confidence for T-test. Possible values: 0 (80%), 1 (90%), 2 (95%), 3 (98%), 4 (99%), 5 (99.5%). Default value: 5. In the [比较模式](#clickhouse-benchmark-comparison-mode) `clickhouse-benchmark` 执行 [独立双样本学生的t测试](https://en.wikipedia.org/wiki/Student%27s_t-test#Independent_two-sample_t-test) 测试以确定两个分布是否与所选置信水平没有不同。 - `--cumulative` — Printing cumulative data instead of data per interval. @@ -51,14 +49,14 @@ clickhouse-benchmark [keys] < queries_file - `--user=USERNAME` — ClickHouse user name. Default value: `default`. - `--password=PSWD` — ClickHouse user password. Default value: empty string. - `--stacktrace` — Stack traces output. When the key is set, `clickhouse-bencmark` 输出异常的堆栈跟踪。 -- `--stage=WORD` — Query processing stage at server. ClickHouse stops query processing and returns answer to `clickhouse-benchmark` 在指定的阶段。 可能的值: `complete`, `fetch_columns`, `with_mergeable_state`. 默认值: `complete`. +- `--stage=WORD` — 查询请求的服务端处理状态. 在特定阶段Clickhouse会停止查询处理，并返回结果给`clickhouse-benchmark`。 可能的值: `complete`, `fetch_columns`, `with_mergeable_state`. 默认值: `complete`. - `--help` — Shows the help message. -如果你想申请一些 [设置](../../operations/settings/index.md) 对于查询，请将它们作为键传递 `--= SETTING_VALUE`. 例如, `--max_memory_usage=1048576`. +如果你想在查询时应用上述的部分参数 [设置](../../operations/settings/index.md) ，请将它们作为键传递 `--= SETTING_VALUE`. 例如, `--max_memory_usage=1048576`. ## 输出 {#clickhouse-benchmark-output} -默认情况下, `clickhouse-benchmark` 每个报表 `--delay` 间隔。 +默认情况下, `clickhouse-benchmark` 按照 `--delay` 参数间隔输出结果。 报告示例: @@ -83,27 +81,27 @@ localhost:9000, queries 10, QPS: 6.772, RPS: 67904487.440, MiB/s: 518.070, resul 99.990% 0.150 sec. ``` -在报告中，您可以找到: +在结果报告中，您可以找到: -- 在查询的数量 `Queries executed:` 场。 +- 查询数量：参见`Queries executed:`字段。 -- 状态字符串包含（按顺序): +- 状态码（按顺序给出）: - - ClickHouse服务器的端点。 + - ClickHouse服务器的连接信息。 - 已处理的查询数。 - - QPS：QPS：在指定的时间段内每秒执行多少个查询服务器 `--delay` 争论。 - - RPS：在指定的时间段内，服务器每秒读取多少行 `--delay` 争论。 - - MiB/s：在指定的时间段内每秒读取多少mebibytes服务器 `--delay` 争论。 - - 结果RPS：在指定的时间段内，服务器每秒放置到查询结果的行数 `--delay` 争论。 - - 结果MiB/s.在指定的时间段内，服务器每秒将多少mebibytes放置到查询结果中 `--delay` 争论。 + - QPS：服务端每秒处理的查询数量 + - RPS：服务器每秒读取多少行 + - MiB/s：服务器每秒读取多少字节的数据 + - 结果RPS：服务端每秒生成多少行的结果集数据 + - 结果MiB/s.服务端每秒生成多少字节的结果集数据 -- 查询执行时间的百分位数。 +- 查询执行时间的百分比。 -## 比较模式 {#clickhouse-benchmark-comparison-mode} +## 对比模式 {#clickhouse-benchmark-comparison-mode} `clickhouse-benchmark` 可以比较两个正在运行的ClickHouse服务器的性能。 -要使用比较模式，请通过以下两对指定两个服务器的端点 `--host`, `--port` 钥匙 键在参数列表中的位置匹配在一起，第一 `--host` 与第一匹配 `--port` 等等。 `clickhouse-benchmark` 建立到两个服务器的连接，然后发送查询。 每个查询寻址到随机选择的服务器。 每个服务器的结果分别显示。 +要使用对比模式，分别为每个服务器配置各自的`--host`, `--port`参数。`clickhouse-benchmark` 会根据设置的参数建立到各个Server的连接并发送请求。每个查询请求会随机发送到某个服务器。输出结果会按服务器分组输出 ## 示例 {#clickhouse-benchmark-example} From 01d903c60d3913584bf4207dbc51563493415349 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Thu, 11 Jun 2020 10:31:37 +0000 Subject: [PATCH 134/329] add minMap and maxMap functions --- .../aggregate-functions/reference.md | 40 +++++++++ .../AggregateFunctionSumMap.cpp | 74 ++++++++++++++++ .../AggregateFunctionSumMap.h | 72 ++++++++++++---- src/Common/FieldVisitors.h | 84 +++++++++++++++++++ .../01280_min_map_max_map.reference | 24 ++++++ .../0_stateless/01280_min_map_max_map.sql | 33 ++++++++ 6 files changed, 310 insertions(+), 17 deletions(-) create mode 100644 tests/queries/0_stateless/01280_min_map_max_map.reference create mode 100644 tests/queries/0_stateless/01280_min_map_max_map.sql diff --git a/docs/en/sql-reference/aggregate-functions/reference.md b/docs/en/sql-reference/aggregate-functions/reference.md index baba1ce904e..5af70ff136b 100644 --- a/docs/en/sql-reference/aggregate-functions/reference.md +++ b/docs/en/sql-reference/aggregate-functions/reference.md @@ -370,6 +370,46 @@ GROUP BY timeslot └─────────────────────┴──────────────────────────────────────────────┴────────────────────────────────┘ ``` +## minMap(key, value), minMap(Tuple(key, value)) {#agg_functions-minmap} + +Calculates the minimum from ‘value’ array according to the keys specified in the ‘key’ array. +Passing tuple of keys and values arrays is synonymical to passing two arrays of keys and values. +The number of elements in ‘key’ and ‘value’ must be the same for each row that is totaled. +Returns a tuple of two arrays: keys in sorted order, and values calculated for the corresponding keys. + +Example: + +```sql +SELECT minMap(a, b) +FROM values('a Array(Int32), b Array(Int64)', ([1, 2], [2, 2]), ([2, 3], [1, 1])) +``` + +```text +┌─minMap(a, b)──────┐ +│ ([1,2,3],[2,1,1]) │ +└───────────────────┘ +``` + +## maxMap(key, value), maxMap(Tuple(key, value)) {#agg_functions-maxmap} + +Calculates the maximum from ‘value’ array according to the keys specified in the ‘key’ array. +Passing tuple of keys and values arrays is synonymical to passing two arrays of keys and values. +The number of elements in ‘key’ and ‘value’ must be the same for each row that is totaled. +Returns a tuple of two arrays: keys in sorted order, and values calculated for the corresponding keys. + +Example: + +```sql +SELECT maxMap(a, b) +FROM values('a Array(Int32), b Array(Int64)', ([1, 2], [2, 2]), ([2, 3], [1, 1])) +``` + +```text +┌─maxMap(a, b)──────┐ +│ ([1,2,3],[2,2,1]) │ +└───────────────────┘ +``` + ## skewPop {#skewpop} Computes the [skewness](https://en.wikipedia.org/wiki/Skewness) of a sequence. diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.cpp b/src/AggregateFunctions/AggregateFunctionSumMap.cpp index f4e299fe7c9..0e0d654abf1 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.cpp +++ b/src/AggregateFunctions/AggregateFunctionSumMap.cpp @@ -32,6 +32,20 @@ struct SumMapFiltered using F = AggregateFunctionSumMapFiltered; }; +template +struct MinMap +{ + template + using F = AggregateFunctionMinMap; +}; + +template +struct MaxMap +{ + template + using F = AggregateFunctionMaxMap; +}; + auto parseArguments(const std::string & name, const DataTypes & arguments) { @@ -154,6 +168,64 @@ AggregateFunctionPtr createAggregateFunctionSumMapFiltered(const std::string & n return res; } +AggregateFunctionPtr createAggregateFunctionMinMap(const std::string & name, const DataTypes & arguments, const Array & params) +{ + assertNoParameters(name, params); + + auto [keys_type, values_types, tuple_argument] = parseArguments(name, arguments); + + AggregateFunctionPtr res; + if (tuple_argument) + { + res.reset(createWithNumericBasedType::template F>(*keys_type, keys_type, values_types, arguments)); + if (!res) + res.reset(createWithDecimalType::template F>(*keys_type, keys_type, values_types, arguments)); + if (!res) + res.reset(createWithStringType::template F>(*keys_type, keys_type, values_types, arguments)); + } + else + { + res.reset(createWithNumericBasedType::template F>(*keys_type, keys_type, values_types, arguments)); + if (!res) + res.reset(createWithDecimalType::template F>(*keys_type, keys_type, values_types, arguments)); + if (!res) + res.reset(createWithStringType::template F>(*keys_type, keys_type, values_types, arguments)); + } + if (!res) + throw Exception("Illegal type of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return res; +} + +AggregateFunctionPtr createAggregateFunctionMaxMap(const std::string & name, const DataTypes & arguments, const Array & params) +{ + assertNoParameters(name, params); + + auto [keys_type, values_types, tuple_argument] = parseArguments(name, arguments); + + AggregateFunctionPtr res; + if (tuple_argument) + { + res.reset(createWithNumericBasedType::template F>(*keys_type, keys_type, values_types, arguments)); + if (!res) + res.reset(createWithDecimalType::template F>(*keys_type, keys_type, values_types, arguments)); + if (!res) + res.reset(createWithStringType::template F>(*keys_type, keys_type, values_types, arguments)); + } + else + { + res.reset(createWithNumericBasedType::template F>(*keys_type, keys_type, values_types, arguments)); + if (!res) + res.reset(createWithDecimalType::template F>(*keys_type, keys_type, values_types, arguments)); + if (!res) + res.reset(createWithStringType::template F>(*keys_type, keys_type, values_types, arguments)); + } + if (!res) + throw Exception("Illegal type of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return res; +} + } void registerAggregateFunctionSumMap(AggregateFunctionFactory & factory) @@ -162,6 +234,8 @@ void registerAggregateFunctionSumMap(AggregateFunctionFactory & factory) factory.registerFunction("sumMapWithOverflow", createAggregateFunctionSumMap); factory.registerFunction("sumMapFiltered", createAggregateFunctionSumMapFiltered); factory.registerFunction("sumMapFilteredWithOverflow", createAggregateFunctionSumMapFiltered); + factory.registerFunction("minMap", createAggregateFunctionMinMap); + factory.registerFunction("maxMap", createAggregateFunctionMaxMap); } } diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index e2aef611955..0c4b407b8a8 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -28,16 +28,16 @@ namespace ErrorCodes } template -struct AggregateFunctionSumMapData +struct AggregateFunctionXxxMapData { // Map needs to be ordered to maintain function properties std::map merged_maps; }; /** Aggregate function, that takes at least two arguments: keys and values, and as a result, builds a tuple of of at least 2 arrays - - * ordered keys and variable number of argument values summed up by corresponding keys. + * ordered keys and variable number of argument values aggregated by corresponding keys. * - * This function is the most useful when using SummingMergeTree to sum Nested columns, which name ends in "Map". + * sumMap function is the most useful when using SummingMergeTree to sum Nested columns, which name ends in "Map". * * Example: sumMap(k, v...) of: * k v @@ -49,25 +49,25 @@ struct AggregateFunctionSumMapData * [8,9,10] [20,20,20] * will return: * ([1,2,3,4,5,6,7,8,9,10],[10,10,45,20,35,20,15,30,20,20]) + * + * minMap and maxMap share the same idea, but calculate min and max correspondingly. */ -template -class AggregateFunctionSumMapBase : public IAggregateFunctionDataHelper< - AggregateFunctionSumMapData>, Derived> +template +class AggregateFunctionMapOpBase : public IAggregateFunctionDataHelper< + AggregateFunctionXxxMapData>, Derived> { private: DataTypePtr keys_type; DataTypes values_types; public: - AggregateFunctionSumMapBase( + AggregateFunctionMapOpBase( const DataTypePtr & keys_type_, const DataTypes & values_types_, const DataTypes & argument_types_, const Array & params_) - : IAggregateFunctionDataHelper>, Derived>(argument_types_, params_) + : IAggregateFunctionDataHelper>, Derived>(argument_types_, params_) , keys_type(keys_type_), values_types(values_types_) {} - String getName() const override { return "sumMap"; } - DataTypePtr getReturnType() const override { DataTypes types; @@ -88,7 +88,7 @@ public: // No overflow, meaning we promote the types if necessary. if (!value_type->canBePromoted()) { - throw Exception{"Values to be summed are expected to be Numeric, Float or Decimal.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception{"Values for " + getName() + " are expected to be Numeric, Float or Decimal.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; } result_type = value_type->promoteNumericType(); @@ -161,7 +161,7 @@ public: if (it != merged_maps.end()) { - applyVisitor(FieldVisitorSum(value), it->second[col]); + applyVisitor(Visitor(value), it->second[col]); } else { @@ -198,7 +198,7 @@ public: if (it != merged_maps.end()) { for (size_t col = 0; col < values_types.size(); ++col) - applyVisitor(FieldVisitorSum(elem.second[col]), it->second[col]); + applyVisitor(Visitor(elem.second[col]), it->second[col]); } else merged_maps[elem.first] = elem.second; @@ -300,15 +300,16 @@ public: } bool keepKey(const T & key) const { return static_cast(*this).keepKey(key); } + String getName() const override { return static_cast(*this).getName(); } }; template class AggregateFunctionSumMap final : - public AggregateFunctionSumMapBase, overflow, tuple_argument> + public AggregateFunctionMapOpBase, FieldVisitorSum, overflow, tuple_argument> { private: using Self = AggregateFunctionSumMap; - using Base = AggregateFunctionSumMapBase; + using Base = AggregateFunctionMapOpBase; public: AggregateFunctionSumMap(const DataTypePtr & keys_type_, DataTypes & values_types_, const DataTypes & argument_types_) @@ -322,14 +323,15 @@ public: template class AggregateFunctionSumMapFiltered final : - public AggregateFunctionSumMapBase, + FieldVisitorSum, overflow, tuple_argument> { private: using Self = AggregateFunctionSumMapFiltered; - using Base = AggregateFunctionSumMapBase; + using Base = AggregateFunctionMapOpBase; std::unordered_set keys_to_keep; @@ -351,4 +353,40 @@ public: bool keepKey(const T & key) const { return keys_to_keep.count(key); } }; +template +class AggregateFunctionMinMap final : + public AggregateFunctionMapOpBase, FieldVisitorMin, true, tuple_argument> +{ +private: + using Self = AggregateFunctionMinMap; + using Base = AggregateFunctionMapOpBase; + +public: + AggregateFunctionMinMap(const DataTypePtr & keys_type_, DataTypes & values_types_, const DataTypes & argument_types_) + : Base{keys_type_, values_types_, argument_types_, {}} + {} + + String getName() const override { return "minMap"; } + + bool keepKey(const T &) const { return true; } +}; + +template +class AggregateFunctionMaxMap final : + public AggregateFunctionMapOpBase, FieldVisitorMax, true, tuple_argument> +{ +private: + using Self = AggregateFunctionMaxMap; + using Base = AggregateFunctionMapOpBase; + +public: + AggregateFunctionMaxMap(const DataTypePtr & keys_type_, DataTypes & values_types_, const DataTypes & argument_types_) + : Base{keys_type_, values_types_, argument_types_, {}} + {} + + String getName() const override { return "maxMap"; } + + bool keepKey(const T &) const { return true; } +}; + } diff --git a/src/Common/FieldVisitors.h b/src/Common/FieldVisitors.h index 90f80974ab1..40a75b162c9 100644 --- a/src/Common/FieldVisitors.h +++ b/src/Common/FieldVisitors.h @@ -449,4 +449,88 @@ public: } }; +/** Implements `Max` operation. + * Returns true if changed + */ +class FieldVisitorMax : public StaticVisitor +{ +private: + const Field & rhs; +public: + explicit FieldVisitorMax(const Field & rhs_) : rhs(rhs_) {} + + bool operator() (Null &) const { throw Exception("Cannot compare Nulls", ErrorCodes::LOGICAL_ERROR); } + bool operator() (Array &) const { throw Exception("Cannot compare Arrays", ErrorCodes::LOGICAL_ERROR); } + bool operator() (Tuple &) const { throw Exception("Cannot compare Tuples", ErrorCodes::LOGICAL_ERROR); } + bool operator() (AggregateFunctionStateData &) const { throw Exception("Cannot compare AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); } + + template + bool operator() (DecimalField & x) const + { + auto val = get>(rhs); + if (val > x) + { + x = val; + return true; + } + + return false; + } + + template + bool operator() (T & x) const + { + auto val = get(rhs); + if (val > x) + { + x = val; + return true; + } + + return false; + } +}; + +/** Implements `Min` operation. + * Returns true if changed + */ +class FieldVisitorMin : public StaticVisitor +{ +private: + const Field & rhs; +public: + explicit FieldVisitorMin(const Field & rhs_) : rhs(rhs_) {} + + bool operator() (Null &) const { throw Exception("Cannot compare Nulls", ErrorCodes::LOGICAL_ERROR); } + bool operator() (Array &) const { throw Exception("Cannot sum Arrays", ErrorCodes::LOGICAL_ERROR); } + bool operator() (Tuple &) const { throw Exception("Cannot sum Tuples", ErrorCodes::LOGICAL_ERROR); } + bool operator() (AggregateFunctionStateData &) const { throw Exception("Cannot sum AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); } + + template + bool operator() (DecimalField & x) const + { + auto val = get>(rhs); + if (val < x) + { + x = val; + return true; + } + + return false; + } + + template + bool operator() (T & x) const + { + auto val = get(rhs); + if (val < x) + { + x = val; + return true; + } + + return false; + } +}; + } diff --git a/tests/queries/0_stateless/01280_min_map_max_map.reference b/tests/queries/0_stateless/01280_min_map_max_map.reference new file mode 100644 index 00000000000..dd707d602c7 --- /dev/null +++ b/tests/queries/0_stateless/01280_min_map_max_map.reference @@ -0,0 +1,24 @@ +([0,1,2,3,4,5,6,7,8,9,10],[10,1,1,1,1,1,1,1,1,1,1]) Tuple(Array(Int32), Array(UInt64)) +([1],[-49]) +([1.00],[-49.00]) +([0,1,2,3,4,5,6,7,8,9,10],[100,91,92,93,94,95,96,97,98,99,1]) Tuple(Array(Int32), Array(UInt64)) +([1],[50]) +([1.00],[50.00]) +(['01234567-89ab-cdef-0123-456789abcdef'],['01111111-89ab-cdef-0123-456789abcdef']) +(['1'],['1']) +(['1'],['1']) +([1],[1]) +([1],[1]) +(['1970-01-02'],[1]) +(['1970-01-01 03:00:01'],[1]) +([1.01],[1]) +(['a'],[1]) +(['01234567-89ab-cdef-0123-456789abcdef'],['02222222-89ab-cdef-0123-456789abcdef']) +(['1'],['2']) +(['1'],['2']) +([1],[2]) +([1],[2]) +(['1970-01-02'],[2]) +(['1970-01-01 03:00:01'],[2]) +([1.01],[2]) +(['a'],[2]) diff --git a/tests/queries/0_stateless/01280_min_map_max_map.sql b/tests/queries/0_stateless/01280_min_map_max_map.sql new file mode 100644 index 00000000000..02731eee601 --- /dev/null +++ b/tests/queries/0_stateless/01280_min_map_max_map.sql @@ -0,0 +1,33 @@ +select minMap([toInt32(number % 10), number % 10 + 1], [number, 1]) as m, toTypeName(m) from numbers(1, 100); +select minMap([1], [toInt32(number) - 50]) from numbers(1, 100); +select minMap([cast(1, 'Decimal(10, 2)')], [cast(toInt32(number) - 50, 'Decimal(10, 2)')]) from numbers(1, 100); + +select maxMap([toInt32(number % 10), number % 10 + 1], [number, 1]) as m, toTypeName(m) from numbers(1, 100); +select maxMap([1], [toInt32(number) - 50]) from numbers(1, 100); +select maxMap([cast(1, 'Decimal(10, 2)')], [cast(toInt32(number) - 50, 'Decimal(10, 2)')]) from numbers(1, 100); + +-- check different types for minMap +select minMap(val, cnt) from values ('val Array(UUID), cnt Array(UUID)', + (['01234567-89ab-cdef-0123-456789abcdef'], ['01111111-89ab-cdef-0123-456789abcdef']), + (['01234567-89ab-cdef-0123-456789abcdef'], ['02222222-89ab-cdef-0123-456789abcdef'])); +select minMap(val, cnt) from values ('val Array(String), cnt Array(String)', (['1'], ['1']), (['1'], ['2'])); +select minMap(val, cnt) from values ('val Array(FixedString(1)), cnt Array(FixedString(1))', (['1'], ['1']), (['1'], ['2'])); +select minMap(val, cnt) from values ('val Array(UInt64), cnt Array(UInt64)', ([1], [1]), ([1], [2])); +select minMap(val, cnt) from values ('val Array(Float64), cnt Array(Int8)', ([1], [1]), ([1], [2])); +select minMap(val, cnt) from values ('val Array(Date), cnt Array(Int16)', ([1], [1]), ([1], [2])); +select minMap(val, cnt) from values ('val Array(DateTime(\'Europe/Moscow\')), cnt Array(Int32)', ([1], [1]), ([1], [2])); +select minMap(val, cnt) from values ('val Array(Decimal(10, 2)), cnt Array(Int16)', (['1.01'], [1]), (['1.01'], [2])); +select minMap(val, cnt) from values ('val Array(Enum16(\'a\'=1)), cnt Array(Int16)', (['a'], [1]), (['a'], [2])); + +-- check different types for maxMap +select maxMap(val, cnt) from values ('val Array(UUID), cnt Array(UUID)', + (['01234567-89ab-cdef-0123-456789abcdef'], ['01111111-89ab-cdef-0123-456789abcdef']), + (['01234567-89ab-cdef-0123-456789abcdef'], ['02222222-89ab-cdef-0123-456789abcdef'])); +select maxMap(val, cnt) from values ('val Array(String), cnt Array(String)', (['1'], ['1']), (['1'], ['2'])); +select maxMap(val, cnt) from values ('val Array(FixedString(1)), cnt Array(FixedString(1))', (['1'], ['1']), (['1'], ['2'])); +select maxMap(val, cnt) from values ('val Array(UInt64), cnt Array(UInt64)', ([1], [1]), ([1], [2])); +select maxMap(val, cnt) from values ('val Array(Float64), cnt Array(Int8)', ([1], [1]), ([1], [2])); +select maxMap(val, cnt) from values ('val Array(Date), cnt Array(Int16)', ([1], [1]), ([1], [2])); +select maxMap(val, cnt) from values ('val Array(DateTime(\'Europe/Moscow\')), cnt Array(Int32)', ([1], [1]), ([1], [2])); +select maxMap(val, cnt) from values ('val Array(Decimal(10, 2)), cnt Array(Int16)', (['1.01'], [1]), (['1.01'], [2])); +select maxMap(val, cnt) from values ('val Array(Enum16(\'a\'=1)), cnt Array(Int16)', (['a'], [1]), (['a'], [2])); From fcfb6d3bc2b0f1a914cfedcc162a9ad5232f93bb Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 13 Jun 2020 11:51:07 +0300 Subject: [PATCH 135/329] Merge with master --- src/Storages/IStorage.cpp | 6 +++--- src/Storages/IStorage.h | 6 +++--- src/Storages/TTLDescription.cpp | 3 +++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index a09bb45f9d0..74c422af385 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -32,13 +32,13 @@ namespace ErrorCodes extern const int DEADLOCK_AVOIDED; } -ColumnsDescription IStorage::getColumns() const +const ColumnsDescription & IStorage::getColumns() const { std::lock_guard lock(metadata_mutex); return metadata.columns; } -IndicesDescription IStorage::getSecondaryIndices() const +const IndicesDescription & IStorage::getSecondaryIndices() const { std::lock_guard lock(metadata_mutex); return metadata.secondary_indices; @@ -51,7 +51,7 @@ bool IStorage::hasSecondaryIndices() const return !metadata.secondary_indices.empty(); } -ConstraintsDescription IStorage::getConstraints() const +const ConstraintsDescription & IStorage::getConstraints() const { std::lock_guard lock(metadata_mutex); return metadata.constraints; diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 40ca901640b..3bacab6f46f 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -142,15 +142,15 @@ public: /// structure lock to get consistent metadata snapshot. This will be fixed /// soon. TODO(alesap) - ColumnsDescription getColumns() const; /// returns combined set of columns + const ColumnsDescription & getColumns() const; /// returns combined set of columns void setColumns(ColumnsDescription columns_); /// sets only real columns, possibly overwrites virtual ones. void setSecondaryIndices(IndicesDescription secondary_indices_); - IndicesDescription getSecondaryIndices() const; + const IndicesDescription & getSecondaryIndices() const; /// Has at least one non primary index bool hasSecondaryIndices() const; - ConstraintsDescription getConstraints() const; + const ConstraintsDescription & getConstraints() const; void setConstraints(ConstraintsDescription constraints_); /// Storage settings diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 898df5006fd..ea6b3e64aff 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -34,6 +34,9 @@ TTLAggregateDescription::TTLAggregateDescription(const TTLAggregateDescription & TTLAggregateDescription & TTLAggregateDescription::operator=(const TTLAggregateDescription & other) { + if (&other == this) + return *this; + column_name = other.column_name; expression_result_column_name = other.expression_result_column_name; if (other.expression) From 31b852c46dd517a8cfe1ebab3c8505ee09f4d481 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 13 Jun 2020 11:53:40 +0300 Subject: [PATCH 136/329] Remove redundant locks --- src/Storages/IStorage.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 74c422af385..9b1f4963dab 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -34,26 +34,21 @@ namespace ErrorCodes const ColumnsDescription & IStorage::getColumns() const { - std::lock_guard lock(metadata_mutex); return metadata.columns; } const IndicesDescription & IStorage::getSecondaryIndices() const { - std::lock_guard lock(metadata_mutex); return metadata.secondary_indices; } - bool IStorage::hasSecondaryIndices() const { - std::lock_guard lock(metadata_mutex); return !metadata.secondary_indices.empty(); } const ConstraintsDescription & IStorage::getConstraints() const { - std::lock_guard lock(metadata_mutex); return metadata.constraints; } @@ -294,7 +289,6 @@ void IStorage::check(const Block & block, bool need_all) const void IStorage::setColumns(ColumnsDescription columns_) { - std::lock_guard lock(metadata_mutex); if (columns_.getOrdinary().empty()) throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); metadata.columns = std::move(columns_); @@ -302,13 +296,11 @@ void IStorage::setColumns(ColumnsDescription columns_) void IStorage::setSecondaryIndices(IndicesDescription secondary_indices_) { - std::lock_guard lock(metadata_mutex); metadata.secondary_indices = std::move(secondary_indices_); } void IStorage::setConstraints(ConstraintsDescription constraints_) { - std::lock_guard lock(metadata_mutex); metadata.constraints = std::move(constraints_); } From d636cdf4b02ae52bc7bddc6eb642401286ba8dc1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 13 Jun 2020 11:55:03 +0300 Subject: [PATCH 137/329] Remove outdated comment --- src/Storages/IStorage.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 3bacab6f46f..d48f269e833 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -137,10 +137,7 @@ public: using ColumnSizeByName = std::unordered_map; virtual ColumnSizeByName getColumnSizes() const { return {}; } -public: - /// NOTE: These methods are thread-safe now, but require additional - /// structure lock to get consistent metadata snapshot. This will be fixed - /// soon. TODO(alesap) +public: /// thread-unsafe part. lockStructure must be acquired const ColumnsDescription & getColumns() const; /// returns combined set of columns void setColumns(ColumnsDescription columns_); /// sets only real columns, possibly overwrites virtual ones. From 18b58e8483a5ee1c0c5be77b6e56950a6087f8ee Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 13 Jun 2020 12:12:45 +0300 Subject: [PATCH 138/329] Revert too strict lock --- src/Storages/IStorage.cpp | 65 +++++++----------------- src/Storages/IStorage.h | 12 ++--- src/Storages/MergeTree/MergeTreeData.cpp | 2 - src/Storages/MergeTree/MergeTreeData.h | 5 -- 4 files changed, 25 insertions(+), 59 deletions(-) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 9b1f4963dab..b81b314c721 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -415,160 +415,138 @@ NamesAndTypesList IStorage::getVirtuals() const return {}; } -KeyDescription IStorage::getPartitionKey() const +const KeyDescription & IStorage::getPartitionKey() const { - std::lock_guard lock(metadata_mutex); return metadata.partition_key; } void IStorage::setPartitionKey(const KeyDescription & partition_key_) { - std::lock_guard lock(metadata_mutex); metadata.partition_key = partition_key_; } bool IStorage::isPartitionKeyDefined() const { - std::lock_guard lock(metadata_mutex); return metadata.partition_key.definition_ast != nullptr; } bool IStorage::hasPartitionKey() const { - std::lock_guard lock(metadata_mutex); return !metadata.partition_key.column_names.empty(); } Names IStorage::getColumnsRequiredForPartitionKey() const { - std::lock_guard lock(metadata_mutex); - if (!metadata.partition_key.column_names.empty()) + if (hasPartitionKey()) return metadata.partition_key.expression->getRequiredColumns(); return {}; } -KeyDescription IStorage::getSortingKey() const +const KeyDescription & IStorage::getSortingKey() const { - std::lock_guard lock(metadata_mutex); return metadata.sorting_key; } void IStorage::setSortingKey(const KeyDescription & sorting_key_) { - std::lock_guard lock(metadata_mutex); metadata.sorting_key = sorting_key_; } bool IStorage::isSortingKeyDefined() const { - std::lock_guard lock(metadata_mutex); return metadata.sorting_key.definition_ast != nullptr; } bool IStorage::hasSortingKey() const { - std::lock_guard lock(metadata_mutex); return !metadata.sorting_key.column_names.empty(); } Names IStorage::getColumnsRequiredForSortingKey() const { - std::lock_guard lock(metadata_mutex); - if (!metadata.sorting_key.column_names.empty()) + if (hasSortingKey()) return metadata.sorting_key.expression->getRequiredColumns(); return {}; } Names IStorage::getSortingKeyColumns() const { - std::lock_guard lock(metadata_mutex); - if (!metadata.sorting_key.column_names.empty()) + if (hasSortingKey()) return metadata.sorting_key.column_names; return {}; } -KeyDescription IStorage::getPrimaryKey() const +const KeyDescription & IStorage::getPrimaryKey() const { - std::lock_guard lock(metadata_mutex); return metadata.primary_key; } void IStorage::setPrimaryKey(const KeyDescription & primary_key_) { - std::lock_guard lock(metadata_mutex); metadata.primary_key = primary_key_; } bool IStorage::isPrimaryKeyDefined() const { - std::lock_guard lock(metadata_mutex); return metadata.primary_key.definition_ast != nullptr; } bool IStorage::hasPrimaryKey() const { - std::lock_guard lock(metadata_mutex); return !metadata.primary_key.column_names.empty(); } Names IStorage::getColumnsRequiredForPrimaryKey() const { - std::lock_guard lock(metadata_mutex); - if (!metadata.primary_key.column_names.empty()) + if (hasPrimaryKey()) return metadata.primary_key.expression->getRequiredColumns(); return {}; } Names IStorage::getPrimaryKeyColumns() const { - std::lock_guard lock(metadata_mutex); if (!metadata.primary_key.column_names.empty()) return metadata.primary_key.column_names; return {}; } -KeyDescription IStorage::getSamplingKey() const +const KeyDescription & IStorage::getSamplingKey() const { - std::lock_guard lock(metadata_mutex); return metadata.sampling_key; } void IStorage::setSamplingKey(const KeyDescription & sampling_key_) { - std::lock_guard lock(metadata_mutex); metadata.sampling_key = sampling_key_; } bool IStorage::isSamplingKeyDefined() const { - std::lock_guard lock(metadata_mutex); return metadata.sampling_key.definition_ast != nullptr; } bool IStorage::hasSamplingKey() const { - std::lock_guard lock(metadata_mutex); return !metadata.sampling_key.column_names.empty(); } Names IStorage::getColumnsRequiredForSampling() const { - std::lock_guard lock(metadata_mutex); - if (!metadata.sampling_key.column_names.empty()) + if (hasSamplingKey()) return metadata.sampling_key.expression->getRequiredColumns(); return {}; } TTLTableDescription IStorage::getTableTTLs() const { - std::lock_guard lock(metadata_mutex); + std::lock_guard lock(ttl_mutex); return metadata.table_ttl; } void IStorage::setTableTTLs(const TTLTableDescription & table_ttl_) { - std::lock_guard lock(metadata_mutex); + std::lock_guard lock(ttl_mutex); metadata.table_ttl = table_ttl_; } @@ -579,43 +557,43 @@ bool IStorage::hasAnyTableTTL() const TTLColumnsDescription IStorage::getColumnTTLs() const { - std::lock_guard lock(metadata_mutex); + std::lock_guard lock(ttl_mutex); return metadata.column_ttls_by_name; } void IStorage::setColumnTTLs(const TTLColumnsDescription & column_ttls_by_name_) { - std::lock_guard lock(metadata_mutex); + std::lock_guard lock(ttl_mutex); metadata.column_ttls_by_name = column_ttls_by_name_; } bool IStorage::hasAnyColumnTTL() const { - std::lock_guard lock(metadata_mutex); + std::lock_guard lock(ttl_mutex); return !metadata.column_ttls_by_name.empty(); } TTLDescription IStorage::getRowsTTL() const { - std::lock_guard lock(metadata_mutex); + std::lock_guard lock(ttl_mutex); return metadata.table_ttl.rows_ttl; } bool IStorage::hasRowsTTL() const { - std::lock_guard lock(metadata_mutex); + std::lock_guard lock(ttl_mutex); return metadata.table_ttl.rows_ttl.expression != nullptr; } TTLDescriptions IStorage::getMoveTTLs() const { - std::lock_guard lock(metadata_mutex); + std::lock_guard lock(ttl_mutex); return metadata.table_ttl.move_ttl; } bool IStorage::hasAnyMoveTTL() const { - std::lock_guard lock(metadata_mutex); + std::lock_guard lock(ttl_mutex); return !metadata.table_ttl.move_ttl.empty(); } @@ -681,7 +659,6 @@ ColumnDependencies IStorage::getColumnDependencies(const NameSet & updated_colum ASTPtr IStorage::getSettingsChanges() const { - std::lock_guard lock(metadata_mutex); if (metadata.settings_changes) return metadata.settings_changes->clone(); return nullptr; @@ -689,28 +666,24 @@ ASTPtr IStorage::getSettingsChanges() const void IStorage::setSettingsChanges(const ASTPtr & settings_changes_) { - std::lock_guard lock(metadata_mutex); if (settings_changes_) metadata.settings_changes = settings_changes_->clone(); else metadata.settings_changes = nullptr; } -SelectQueryDescription IStorage::getSelectQuery() const +const SelectQueryDescription & IStorage::getSelectQuery() const { - std::lock_guard lock(metadata_mutex); return metadata.select; } void IStorage::setSelectQuery(const SelectQueryDescription & select_) { - std::lock_guard lock(metadata_mutex); metadata.select = select_; } bool IStorage::hasSelectQuery() const { - std::lock_guard lock(metadata_mutex); return metadata.select.select_query != nullptr; } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index d48f269e833..c7c8e382a87 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -156,7 +156,7 @@ public: /// thread-unsafe part. lockStructure must be acquired bool hasSettingsChanges() const { return metadata.settings_changes != nullptr; } /// Select query for *View storages. - SelectQueryDescription getSelectQuery() const; + const SelectQueryDescription & getSelectQuery() const; void setSelectQuery(const SelectQueryDescription & select_); bool hasSelectQuery() const; @@ -206,7 +206,7 @@ private: mutable std::mutex id_mutex; /// TODO (alesap) just use multiversion for atomic metadata - mutable std::mutex metadata_mutex; + mutable std::mutex ttl_mutex; StorageInMemoryMetadata metadata; private: RWLockImpl::LockHolder tryLockTimed( @@ -440,7 +440,7 @@ public: virtual Strings getDataPaths() const { return {}; } /// Returns structure with partition key. - KeyDescription getPartitionKey() const; + const KeyDescription & getPartitionKey() const; /// Set partition key for storage (methods bellow, are just wrappers for this /// struct). void setPartitionKey(const KeyDescription & partition_key_); @@ -455,7 +455,7 @@ public: /// Returns structure with sorting key. - KeyDescription getSortingKey() const; + const KeyDescription & getSortingKey() const; /// Set sorting key for storage (methods bellow, are just wrappers for this /// struct). void setSortingKey(const KeyDescription & sorting_key_); @@ -472,7 +472,7 @@ public: Names getSortingKeyColumns() const; /// Returns structure with primary key. - KeyDescription getPrimaryKey() const; + const KeyDescription & getPrimaryKey() const; /// Set primary key for storage (methods bellow, are just wrappers for this /// struct). void setPrimaryKey(const KeyDescription & primary_key_); @@ -490,7 +490,7 @@ public: Names getPrimaryKeyColumns() const; /// Returns structure with sampling key. - KeyDescription getSamplingKey() const; + const KeyDescription & getSamplingKey() const; /// Set sampling key for storage (methods bellow, are just wrappers for this /// struct). void setSamplingKey(const KeyDescription & sampling_key_); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index fdd62b03046..84470088ebe 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -529,7 +529,6 @@ void MergeTreeData::setTTLExpressions(const StorageInMemoryMetadata & new_metada { checkTTLExpressions(new_metadata); setColumnTTLs(new_metadata.column_ttls_by_name); - auto move_ttl_entries_lock = std::lock_guard(move_ttl_entries_mutex); setTableTTLs(new_metadata.table_ttl); } @@ -2807,7 +2806,6 @@ MergeTreeData::selectTTLEntryForTTLInfos(const IMergeTreeDataPart::TTLInfos & tt time_t max_max_ttl = 0; TTLDescriptions::const_iterator best_entry_it; - auto lock = std::lock_guard(move_ttl_entries_mutex); const auto & move_ttl_entries = getMoveTTLs(); for (auto ttl_entry_it = move_ttl_entries.begin(); ttl_entry_it != move_ttl_entries.end(); ++ttl_entry_it) { diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index a101e0645e3..007c6898e60 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -640,11 +640,6 @@ public: std::optional selectTTLEntryForTTLInfos(const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const; - /// This mutex is required for background move operations which do not - /// obtain global locks. - /// TODO (alesap) It will be removed after metadata became atomic - mutable std::mutex move_ttl_entries_mutex; - /// Limiting parallel sends per one table, used in DataPartsExchange std::atomic_uint current_table_sends {0}; From 3427da1c43e283100424a452aad828975422581c Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 13 Jun 2020 13:39:25 +0300 Subject: [PATCH 139/329] Fix incorrect usage of rows TTL --- src/DataStreams/TTLBlockInputStream.cpp | 36 ++++++++++++++----------- src/Storages/IStorage.cpp | 3 ++- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp index ca65ae520c6..c79abff98cd 100644 --- a/src/DataStreams/TTLBlockInputStream.cpp +++ b/src/DataStreams/TTLBlockInputStream.cpp @@ -70,21 +70,22 @@ TTLBlockInputStream::TTLBlockInputStream( defaults_expression = ExpressionAnalyzer{default_expr_list, syntax_result, storage.global_context}.getActions(true); } - if (storage.hasRowsTTL() && storage.getRowsTTL().mode == TTLMode::GROUP_BY) + auto storage_rows_ttl = storage.getRowsTTL(); + if (storage.hasRowsTTL() && storage_rows_ttl.mode == TTLMode::GROUP_BY) { - current_key_value.resize(storage.getRowsTTL().group_by_keys.size()); + current_key_value.resize(storage_rows_ttl.group_by_keys.size()); ColumnNumbers keys; - for (const auto & key : storage.getRowsTTL().group_by_keys) + for (const auto & key : storage_rows_ttl.group_by_keys) keys.push_back(header.getPositionByName(key)); - agg_key_columns.resize(storage.getRowsTTL().group_by_keys.size()); + agg_key_columns.resize(storage_rows_ttl.group_by_keys.size()); - AggregateDescriptions aggregates = storage.getRowsTTL().aggregate_descriptions; + AggregateDescriptions aggregates = storage_rows_ttl.aggregate_descriptions; for (auto & descr : aggregates) if (descr.arguments.empty()) for (const auto & name : descr.argument_names) descr.arguments.push_back(header.getPositionByName(name)); - agg_aggregate_columns.resize(storage.getRowsTTL().aggregate_descriptions.size()); + agg_aggregate_columns.resize(storage_rows_ttl.aggregate_descriptions.size()); const Settings & settings = storage.global_context.getSettingsRef(); @@ -105,8 +106,9 @@ bool TTLBlockInputStream::isTTLExpired(time_t ttl) const Block TTLBlockInputStream::readImpl() { /// Skip all data if table ttl is expired for part - if (storage.hasRowsTTL() && !storage.getRowsTTL().where_expression && - storage.getRowsTTL().mode != TTLMode::GROUP_BY && isTTLExpired(old_ttl_infos.table_ttl.max)) + auto storage_rows_ttl = storage.getRowsTTL(); + if (storage.hasRowsTTL() && !storage_rows_ttl.where_expression && + storage_rows_ttl.mode != TTLMode::GROUP_BY && isTTLExpired(old_ttl_infos.table_ttl.max)) { rows_removed = data_part->rows_count; return {}; @@ -151,7 +153,7 @@ void TTLBlockInputStream::readSuffixImpl() void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block) { - const auto & rows_ttl = storage.getRowsTTL(); + auto rows_ttl = storage.getRowsTTL(); rows_ttl.expression->execute(block); if (rows_ttl.where_expression) @@ -160,8 +162,8 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block) const IColumn * ttl_column = block.getByName(rows_ttl.result_column).column.get(); - const IColumn * where_result_column = storage.getRowsTTL().where_expression ? - block.getByName(storage.getRowsTTL().where_result_column).column.get() : nullptr; + const IColumn * where_result_column = rows_ttl.where_expression ? + block.getByName(rows_ttl.where_result_column).column.get() : nullptr; const auto & column_names = header.getNames(); @@ -199,6 +201,7 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block) size_t rows_aggregated = 0; size_t current_key_start = 0; size_t rows_with_current_key = 0; + auto storage_rows_ttl = storage.getRowsTTL(); for (size_t i = 0; i < block.rows(); ++i) { UInt32 cur_ttl = getTimestampByIndex(ttl_column, i); @@ -206,9 +209,9 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block) bool ttl_expired = isTTLExpired(cur_ttl) && where_filter_passed; bool same_as_current = true; - for (size_t j = 0; j < storage.getRowsTTL().group_by_keys.size(); ++j) + for (size_t j = 0; j < storage_rows_ttl.group_by_keys.size(); ++j) { - const String & key_column = storage.getRowsTTL().group_by_keys[j]; + const String & key_column = storage_rows_ttl.group_by_keys[j]; const IColumn * values_column = block.getByName(key_column).column.get(); if (!same_as_current || (*values_column)[i] != current_key_value[j]) { @@ -275,17 +278,18 @@ void TTLBlockInputStream::finalizeAggregates(MutableColumns & result_columns) if (!agg_result.empty()) { auto aggregated_res = aggregator->convertToBlocks(agg_result, true, 1); + auto storage_rows_ttl = storage.getRowsTTL(); for (auto & agg_block : aggregated_res) { - for (const auto & it : storage.getRowsTTL().set_parts) + for (const auto & it : storage_rows_ttl.set_parts) it.expression->execute(agg_block); - for (const auto & name : storage.getRowsTTL().group_by_keys) + for (const auto & name : storage_rows_ttl.group_by_keys) { const IColumn * values_column = agg_block.getByName(name).column.get(); auto & result_column = result_columns[header.getPositionByName(name)]; result_column->insertRangeFrom(*values_column, 0, agg_block.rows()); } - for (const auto & it : storage.getRowsTTL().set_parts) + for (const auto & it : storage_rows_ttl.set_parts) { const IColumn * values_column = agg_block.getByName(it.expression_result_column_name).column.get(); auto & result_column = result_columns[header.getPositionByName(it.column_name)]; diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index b81b314c721..a244f836f5c 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -629,7 +629,8 @@ ColumnDependencies IStorage::getColumnDependencies(const NameSet & updated_colum if (hasRowsTTL()) { - if (add_dependent_columns(getRowsTTL().expression, required_ttl_columns)) + auto rows_expression = getRowsTTL().expression; + if (add_dependent_columns(rows_expression, required_ttl_columns)) { /// Filter all columns, if rows TTL expression have to be recalculated. for (const auto & column : getColumns().getAllPhysical()) From 901a657417beb4d262a86c61b6935210eb9e7893 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Jun 2020 14:20:24 +0300 Subject: [PATCH 140/329] Rename 01307_data_skip_bloom_filter to 01307_bloom_filter_index_string_multi_granulas This better reflects the covered case. --- ...e => 01307_bloom_filter_index_string_multi_granulas.reference} | 0 ...ter.sql => 01307_bloom_filter_index_string_multi_granulas.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{01307_data_skip_bloom_filter.reference => 01307_bloom_filter_index_string_multi_granulas.reference} (100%) rename tests/queries/0_stateless/{01307_data_skip_bloom_filter.sql => 01307_bloom_filter_index_string_multi_granulas.sql} (100%) diff --git a/tests/queries/0_stateless/01307_data_skip_bloom_filter.reference b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.reference similarity index 100% rename from tests/queries/0_stateless/01307_data_skip_bloom_filter.reference rename to tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.reference diff --git a/tests/queries/0_stateless/01307_data_skip_bloom_filter.sql b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql similarity index 100% rename from tests/queries/0_stateless/01307_data_skip_bloom_filter.sql rename to tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql From 9901e4d528bf0ebb3594150116077a93666ec450 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jun 2020 20:20:42 +0300 Subject: [PATCH 141/329] Remove debug output #11554 --- src/Functions/extractAllGroups.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/extractAllGroups.h b/src/Functions/extractAllGroups.h index b75e54b490e..8216a528b2c 100644 --- a/src/Functions/extractAllGroups.h +++ b/src/Functions/extractAllGroups.h @@ -227,7 +227,6 @@ public: row_offset = next_row_offset; } } - DUMP(Kind, needle, column_haystack, root_offsets_col, nested_offsets_col); ColumnArray::MutablePtr nested_array_col = ColumnArray::create(std::move(data_col), std::move(nested_offsets_col)); ColumnArray::MutablePtr root_array_col = ColumnArray::create(std::move(nested_array_col), std::move(root_offsets_col)); From d8312d0f0051fd4523716924a5bd8951e5276bb9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jun 2020 20:25:13 +0300 Subject: [PATCH 142/329] Remove "fail" from test name; fix typo in test name #11151 --- ... => 01281_unsucceeded_insert_select_queries_counter.reference} | 0 ...er.sql => 01281_unsucceeded_insert_select_queries_counter.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{01281_failed_insert_select_queries_couner.reference => 01281_unsucceeded_insert_select_queries_counter.reference} (100%) rename tests/queries/0_stateless/{01281_failed_insert_select_queries_couner.sql => 01281_unsucceeded_insert_select_queries_counter.sql} (100%) diff --git a/tests/queries/0_stateless/01281_failed_insert_select_queries_couner.reference b/tests/queries/0_stateless/01281_unsucceeded_insert_select_queries_counter.reference similarity index 100% rename from tests/queries/0_stateless/01281_failed_insert_select_queries_couner.reference rename to tests/queries/0_stateless/01281_unsucceeded_insert_select_queries_counter.reference diff --git a/tests/queries/0_stateless/01281_failed_insert_select_queries_couner.sql b/tests/queries/0_stateless/01281_unsucceeded_insert_select_queries_counter.sql similarity index 100% rename from tests/queries/0_stateless/01281_failed_insert_select_queries_couner.sql rename to tests/queries/0_stateless/01281_unsucceeded_insert_select_queries_counter.sql From 5f82cc0021bc04177ab2e34e8f4709e61a1a9744 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Jun 2020 16:39:21 +0300 Subject: [PATCH 143/329] Fix description for LoadBalancing::IN_ORDER --- src/Core/SettingsCollection.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/SettingsCollection.h b/src/Core/SettingsCollection.h index 1fe5762de4c..aed8dc6a929 100644 --- a/src/Core/SettingsCollection.h +++ b/src/Core/SettingsCollection.h @@ -225,7 +225,8 @@ enum class LoadBalancing /// a replica is selected among the replicas with the minimum number of errors /// with the minimum number of distinguished characters in the replica name and local hostname NEAREST_HOSTNAME, - /// replicas are walked through strictly in order; the number of errors does not matter + // replicas with the same number of errors are accessed in the same order + // as they are specified in the configuration. IN_ORDER, /// if first replica one has higher number of errors, /// pick a random one from replicas with minimum number of errors From 2cd82a25f52b684e6fc13010f8efe0965c1e94dd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Jun 2020 23:59:20 +0300 Subject: [PATCH 144/329] Remove trivial count query optimization if row-level security is set #11352 --- src/Interpreters/ExpressionAnalyzer.h | 2 ++ src/Interpreters/InterpreterSelectQuery.cpp | 17 ++++++++++------- src/Interpreters/InterpreterSelectQuery.h | 4 +++- .../ReplicatedMergeTreeBlockOutputStream.cpp | 3 ++- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index ed07ab3fe36..c69cb61162f 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -208,7 +208,9 @@ struct ExpressionAnalysisResult const FilterInfoPtr & filter_info, const Block & source_header); + /// Filter for row-level security. bool hasFilter() const { return filter_info.get(); } + bool hasJoin() const { return before_join.get(); } bool hasPrewhere() const { return prewhere_info.get(); } bool hasWhere() const { return before_where.get(); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index f9072e6176a..98cf36cc30b 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -94,7 +94,8 @@ namespace ErrorCodes } /// Assumes `storage` is set and the table filter (row-level security) is not empty. -String InterpreterSelectQuery::generateFilterActions(ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns) const +String InterpreterSelectQuery::generateFilterActions( + ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns) const { const auto & db_name = table_id.getDatabaseName(); const auto & table_name = table_id.getTableName(); @@ -474,8 +475,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl() second_stage, options.only_analyze, filter_info, - source_header - ); + source_header); if (options.to_stage == QueryProcessingStage::Enum::FetchColumns) { @@ -979,10 +979,13 @@ void InterpreterSelectQuery::executeFetchColumns( /// Optimization for trivial query like SELECT count() FROM table. bool optimize_trivial_count = - syntax_analyzer_result->optimize_trivial_count && storage && - processing_stage == QueryProcessingStage::FetchColumns && - query_analyzer->hasAggregation() && (query_analyzer->aggregates().size() == 1) && - typeid_cast(query_analyzer->aggregates()[0].function.get()); + syntax_analyzer_result->optimize_trivial_count + && storage + && !filter_info + && processing_stage == QueryProcessingStage::FetchColumns + && query_analyzer->hasAggregation() + && (query_analyzer->aggregates().size() == 1) + && typeid_cast(query_analyzer->aggregates()[0].function.get()); if (optimize_trivial_count) { diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 34d255e398e..c60451d5f4a 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -132,7 +132,8 @@ private: void executeSubqueriesInSetsAndJoins(QueryPipeline & pipeline, const std::unordered_map & subqueries_for_sets); void executeMergeSorted(QueryPipeline & pipeline, const SortDescription & sort_description, UInt64 limit); - String generateFilterActions(ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns = {}) const; + String generateFilterActions( + ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns = {}) const; enum class Modificator { @@ -159,6 +160,7 @@ private: /// Is calculated in getSampleBlock. Is used later in readImpl. ExpressionAnalysisResult analysis_result; + /// For row-level security. FilterInfoPtr filter_info; QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index c67ea11f56f..1bbc56d940d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -198,7 +198,8 @@ void ReplicatedMergeTreeBlockOutputStream::writeExistingPart(MergeTreeData::Muta } -void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const String & block_id) +void ReplicatedMergeTreeBlockOutputStream::commitPart( + zkutil::ZooKeeperPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const String & block_id) { storage.check(part->getColumns()); assertSessionIsNotExpired(zookeeper); From a421e7e4b40155ddaaab72d0269a68db4f289020 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 00:13:52 +0300 Subject: [PATCH 145/329] Added a test --- programs/server/users.d/access_management.xml | 7 +++++++ ...1308_row_policy_and_trivial_count_query.reference | 3 +++ .../01308_row_policy_and_trivial_count_query.sql | 12 ++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 programs/server/users.d/access_management.xml create mode 100644 tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.reference create mode 100644 tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql diff --git a/programs/server/users.d/access_management.xml b/programs/server/users.d/access_management.xml new file mode 100644 index 00000000000..7e799cb7b10 --- /dev/null +++ b/programs/server/users.d/access_management.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.reference b/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.reference new file mode 100644 index 00000000000..61150aca43c --- /dev/null +++ b/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.reference @@ -0,0 +1,3 @@ +3 +2 +3 diff --git a/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql b/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql new file mode 100644 index 00000000000..c105885cb60 --- /dev/null +++ b/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS t; + +CREATE TABLE t (x UInt8) ENGINE = MergeTree ORDER BY x; +INSERT INTO t VALUES (1), (2), (3); + +SELECT count() FROM t; +CREATE ROW POLICY filter ON t USING (x % 2 = 1) TO ALL; +SELECT count() FROM t; +DROP ROW POLICY filter ON t; +SELECT count() FROM t; + +DROP TABLE t; From c139a05370bf1656f5f75e4e35021683bcf72c37 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Jun 2020 19:31:28 +0300 Subject: [PATCH 146/329] Forward declaration in StorageDistributed --- src/Storages/Distributed/DirectoryMonitor.cpp | 1 + src/Storages/StorageDistributed.cpp | 2 ++ src/Storages/StorageDistributed.h | 6 ++++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index a491cc411b1..8b45573464f 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index d80fee1e4dc..ec69a75e1ba 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -46,6 +47,7 @@ #include #include +#include #include diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 4067012c449..c78097d8abe 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -7,8 +7,6 @@ #include #include #include -#include -#include #include #include #include @@ -17,6 +15,7 @@ namespace DB { +struct Settings; class Context; class VolumeJBOD; @@ -25,6 +24,9 @@ using VolumeJBODPtr = std::shared_ptr; class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; +class Cluster; +using ClusterPtr = std::shared_ptr; + /** A distributed table that resides on multiple servers. * Uses data from the specified database and tables on each server. * From 0e378590fe5ecf7091730e019d78854ab87ef359 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Jun 2020 20:23:51 +0300 Subject: [PATCH 147/329] Add load_balancing=round_robin --- docs/en/operations/settings/settings.md | 9 +++++++++ src/Client/ConnectionPoolWithFailover.cpp | 13 ++++++++++++- src/Client/ConnectionPoolWithFailover.h | 1 + src/Core/SettingsCollection.cpp | 3 ++- src/Core/SettingsCollection.h | 2 ++ 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f29866d4980..3ed6e240f29 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -822,6 +822,7 @@ ClickHouse supports the following algorithms of choosing replicas: - [Nearest hostname](#load_balancing-nearest_hostname) - [In order](#load_balancing-in_order) - [First or random](#load_balancing-first_or_random) +- [Round robin](#load_balancing-round_robin) ### Random (by Default) {#load_balancing-random} @@ -865,6 +866,14 @@ This algorithm chooses the first replica in the set or a random replica if the f The `first_or_random` algorithm solves the problem of the `in_order` algorithm. With `in_order`, if one replica goes down, the next one gets a double load while the remaining replicas handle the usual amount of traffic. When using the `first_or_random` algorithm, the load is evenly distributed among replicas that are still available. +### Round robin {#load_balancing-round_robin} + +``` sql +load_balancing = round_robin +``` + +This algorithm uses round robin policy across replicas with the same number of errors (only the queries with `round_robin` policy is accounted). + ## prefer\_localhost\_replica {#settings-prefer-localhost-replica} Enables/disables preferable using the localhost replica when processing distributed queries. diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 713bb33342f..257f4d36c33 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -11,7 +11,6 @@ #include - namespace ProfileEvents { extern const Event DistributedConnectionMissingTable; @@ -71,6 +70,12 @@ IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts case LoadBalancing::FIRST_OR_RANDOM: get_priority = [](size_t i) -> size_t { return i >= 1; }; break; + case LoadBalancing::ROUND_ROBIN: + if (last_used >= nested_pools.size()) + last_used = 0; + ++last_used; + get_priority = [&](size_t i) { ++i; return i < last_used ? nested_pools.size() - i : i - last_used; }; + break; } return Base::get(try_get_entry, get_priority); @@ -181,6 +186,12 @@ std::vector ConnectionPoolWithFailover::g case LoadBalancing::FIRST_OR_RANDOM: get_priority = [](size_t i) -> size_t { return i >= 1; }; break; + case LoadBalancing::ROUND_ROBIN: + if (last_used >= nested_pools.size()) + last_used = 0; + ++last_used; + get_priority = [&](size_t i) { ++i; return i < last_used ? nested_pools.size() - i : i - last_used; }; + break; } bool fallback_to_stale_replicas = settings ? bool(settings->fallback_to_stale_replicas_for_distributed_queries) : true; diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index bdc06656ff1..10dea98c8f7 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -97,6 +97,7 @@ private: private: std::vector hostname_differences; /// Distances from name of this host to the names of hosts of pools. + size_t last_used = 0; /// Last used for round_robin policy. LoadBalancing default_load_balancing; }; diff --git a/src/Core/SettingsCollection.cpp b/src/Core/SettingsCollection.cpp index 324ad889a65..b36884fad22 100644 --- a/src/Core/SettingsCollection.cpp +++ b/src/Core/SettingsCollection.cpp @@ -481,7 +481,8 @@ void SettingURI::deserialize(ReadBuffer & buf, SettingsBinaryFormat) M(RANDOM, "random") \ M(NEAREST_HOSTNAME, "nearest_hostname") \ M(IN_ORDER, "in_order") \ - M(FIRST_OR_RANDOM, "first_or_random") + M(FIRST_OR_RANDOM, "first_or_random") \ + M(ROUND_ROBIN, "round_robin") IMPLEMENT_SETTING_ENUM(LoadBalancing, LOAD_BALANCING_LIST_OF_NAMES, ErrorCodes::UNKNOWN_LOAD_BALANCING) diff --git a/src/Core/SettingsCollection.h b/src/Core/SettingsCollection.h index aed8dc6a929..71a308fb37e 100644 --- a/src/Core/SettingsCollection.h +++ b/src/Core/SettingsCollection.h @@ -231,6 +231,8 @@ enum class LoadBalancing /// if first replica one has higher number of errors, /// pick a random one from replicas with minimum number of errors FIRST_OR_RANDOM, + // round robin across replicas with the same number of errors. + ROUND_ROBIN, }; using SettingLoadBalancing = SettingEnum; From 9386478a77f5822f074da5742ac287c0b6acc41a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Jun 2020 21:33:58 +0300 Subject: [PATCH 148/329] Add test for load_balancing algorithms --- .../__init__.py | 0 .../configs/remote_servers.xml | 36 ++++++ .../test_distributed_load_balancing/test.py | 114 ++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 tests/integration/test_distributed_load_balancing/__init__.py create mode 100644 tests/integration/test_distributed_load_balancing/configs/remote_servers.xml create mode 100644 tests/integration/test_distributed_load_balancing/test.py diff --git a/tests/integration/test_distributed_load_balancing/__init__.py b/tests/integration/test_distributed_load_balancing/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_distributed_load_balancing/configs/remote_servers.xml b/tests/integration/test_distributed_load_balancing/configs/remote_servers.xml new file mode 100644 index 00000000000..9efd681e74e --- /dev/null +++ b/tests/integration/test_distributed_load_balancing/configs/remote_servers.xml @@ -0,0 +1,36 @@ + + + + + + n1 + 9000 + + + n2 + 9000 + + + n3 + 9000 + + + + + + + n1 + 9000 + + + n2 + 9000 + + + n3 + 9000 + + + + + diff --git a/tests/integration/test_distributed_load_balancing/test.py b/tests/integration/test_distributed_load_balancing/test.py new file mode 100644 index 00000000000..fa6dfb20a88 --- /dev/null +++ b/tests/integration/test_distributed_load_balancing/test.py @@ -0,0 +1,114 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name +# pylint: disable=line-too-long + +import uuid +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +n1 = cluster.add_instance('n1', main_configs=['configs/remote_servers.xml']) +n2 = cluster.add_instance('n2', main_configs=['configs/remote_servers.xml']) +n3 = cluster.add_instance('n3', main_configs=['configs/remote_servers.xml']) + +nodes = len(cluster.instances) +queries = nodes*5 + +def create_tables(): + for n in cluster.instances.values(): + n.query('DROP TABLE IF EXISTS data') + n.query('DROP TABLE IF EXISTS dist') + n.query('CREATE TABLE data (key Int) Engine=Memory()') + n.query(""" + CREATE TABLE dist AS data + Engine=Distributed( + replicas_cluster, + currentDatabase(), + data) + """.format()) + +def make_uuid(): + return uuid.uuid4().hex + +@pytest.fixture(scope='module', autouse=True) +def start_cluster(): + try: + cluster.start() + create_tables() + yield cluster + finally: + cluster.shutdown() + +def get_node(query_node, *args, **kwargs): + query_id = make_uuid() + + settings = { + 'query_id': query_id, + 'log_queries': 1, + 'log_queries_min_type': 'QUERY_START', + 'prefer_localhost_replica': 0, + } + if 'settings' not in kwargs: + kwargs['settings'] = settings + else: + kwargs['settings'].update(settings) + + query_node.query('SELECT * FROM dist', *args, **kwargs) + + for n in cluster.instances.values(): + n.query('SYSTEM FLUSH LOGS') + + rows = query_node.query(""" + SELECT c.host_name + FROM ( + SELECT _shard_num + FROM cluster(shards_cluster, system.query_log) + WHERE + initial_query_id = '{query_id}' AND + is_initial_query = 0 AND + type = 'QueryFinish' + ORDER BY event_date DESC, event_time DESC + LIMIT 1 + ) a + JOIN system.clusters c + ON a._shard_num = c.shard_num AND cluster = 'shards_cluster' + """.format(query_id=query_id)) + return rows.strip() + +# TODO: right now random distribution looks bad, but works +def test_load_balancing_default(): + unique_nodes = set() + for _ in range(0, queries): + unique_nodes.add(get_node(n1, settings={'load_balancing': 'random'})) + assert len(unique_nodes) == nodes, unique_nodes + +def test_load_balancing_nearest_hostname(): + unique_nodes = set() + for _ in range(0, queries): + unique_nodes.add(get_node(n1, settings={'load_balancing': 'nearest_hostname'})) + assert len(unique_nodes) == 1, unique_nodes + assert unique_nodes == set(['n1']) + +def test_load_balancing_in_order(): + unique_nodes = set() + for _ in range(0, queries): + unique_nodes.add(get_node(n1, settings={'load_balancing': 'in_order'})) + assert len(unique_nodes) == 1, unique_nodes + assert unique_nodes == set(['n1']) + +def test_load_balancing_first_or_random(): + unique_nodes = set() + for _ in range(0, queries): + unique_nodes.add(get_node(n1, settings={'load_balancing': 'first_or_random'})) + assert len(unique_nodes) == 1, unique_nodes + assert unique_nodes == set(['n1']) + +# TODO: last_used will be reset on config reload, hence may fail +def test_load_balancing_round_robin(): + unique_nodes = set() + for _ in range(0, nodes): + unique_nodes.add(get_node(n1, settings={'load_balancing': 'round_robin'})) + assert len(unique_nodes) == nodes, unique_nodes + assert unique_nodes == set(['n1', 'n2', 'n3']) From 844140467e6fdb2be464dd48b2153994fbdc0e5c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 01:18:48 +0300 Subject: [PATCH 149/329] Simplify the code in comparison functions --- src/Functions/FunctionsComparison.h | 187 ++++++---------------------- 1 file changed, 41 insertions(+), 146 deletions(-) diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 642995974b5..12da4c772d1 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -812,94 +813,49 @@ private: } } - bool executeDateOrDateTimeOrEnumOrUUIDWithConstString( + bool executeWithConstString( Block & block, size_t result, const IColumn * col_left_untyped, const IColumn * col_right_untyped, - const DataTypePtr & left_type, const DataTypePtr & right_type, bool left_is_num, size_t input_rows_count) + const DataTypePtr & left_type, const DataTypePtr & right_type, size_t input_rows_count) { - /// This is no longer very special case - comparing dates, datetimes, and enumerations with a string constant. - const IColumn * column_string_untyped = !left_is_num ? col_left_untyped : col_right_untyped; - const IColumn * column_number = left_is_num ? col_left_untyped : col_right_untyped; - const IDataType * number_type = left_is_num ? left_type.get() : right_type.get(); + /// To compare something with const string, we cast constant to appropriate type and compare as usual. + /// We should deal with possible overflows, e.g. toUInt8(1) = '257' should return false. - WhichDataType which(number_type); + const ColumnConst * left_const = checkAndGetColumnConstStringOrFixedString(col_left_untyped); + const ColumnConst * right_const = checkAndGetColumnConstStringOrFixedString(col_right_untyped); - const bool legal_types = which.isDateOrDateTime() || which.isEnum() || which.isUUID(); - - const auto column_string = checkAndGetColumnConst(column_string_untyped); - if (!column_string || !legal_types) + if (!left_const && !right_const) return false; - StringRef string_value = column_string->getDataAt(0); + const IDataType * type_string = left_const ? left_type.get() : right_type.get(); + const DataTypePtr & type_to_compare = !left_const ? left_type : right_type; - if (which.isDate()) + Field string_value = left_const ? left_const->getField() : right_const->getField(); + Field converted = convertFieldToType(string_value, *type_to_compare, type_string); + + /// If not possible to convert, comparison yields to false. + if (converted.isNull()) { - DayNum date; - ReadBufferFromMemory in(string_value.data, string_value.size); - readDateText(date, in); - if (!in.eof()) - throw Exception("String is too long for Date: " + string_value.toString(), ErrorCodes::TOO_LARGE_STRING_SIZE); - - ColumnPtr parsed_const_date_holder = DataTypeDate().createColumnConst(input_rows_count, date); - const ColumnConst * parsed_const_date = assert_cast(parsed_const_date_holder.get()); - executeNumLeftType(block, result, - left_is_num ? col_left_untyped : parsed_const_date, - left_is_num ? parsed_const_date : col_right_untyped); + block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count, 0); } - else if (which.isDateTime()) + else { - time_t date_time; - ReadBufferFromMemory in(string_value.data, string_value.size); - readDateTimeText(date_time, in, dynamic_cast(*number_type).getTimeZone()); - if (!in.eof()) - throw Exception("String is too long for DateTime: " + string_value.toString(), ErrorCodes::TOO_LARGE_STRING_SIZE); + auto column_converted = type_to_compare->createColumnConst(input_rows_count, converted); - ColumnPtr parsed_const_date_time_holder = DataTypeDateTime().createColumnConst(input_rows_count, UInt64(date_time)); - const ColumnConst * parsed_const_date_time = assert_cast(parsed_const_date_time_holder.get()); - executeNumLeftType(block, result, - left_is_num ? col_left_untyped : parsed_const_date_time, - left_is_num ? parsed_const_date_time : col_right_untyped); + Block tmp_block + { + { left_const ? column_converted : col_left_untyped->getPtr(), type_to_compare, "" }, + { !left_const ? column_converted : col_right_untyped->getPtr(), type_to_compare, "" }, + block.getByPosition(result) + }; + + executeImpl(tmp_block, {0, 1}, 2, input_rows_count); + + block.getByPosition(result).column = std::move(tmp_block.getByPosition(2).column); } - else if (which.isUUID()) - { - UUID uuid; - ReadBufferFromMemory in(string_value.data, string_value.size); - readText(uuid, in); - if (!in.eof()) - throw Exception("String is too long for UUID: " + string_value.toString(), ErrorCodes::TOO_LARGE_STRING_SIZE); - - ColumnPtr parsed_const_uuid_holder = DataTypeUUID().createColumnConst(input_rows_count, uuid); - const ColumnConst * parsed_const_uuid = assert_cast(parsed_const_uuid_holder.get()); - executeNumLeftType(block, result, - left_is_num ? col_left_untyped : parsed_const_uuid, - left_is_num ? parsed_const_uuid : col_right_untyped); - } - - else if (which.isEnum8()) - executeEnumWithConstString(block, result, column_number, column_string, - number_type, left_is_num, input_rows_count); - else if (which.isEnum16()) - executeEnumWithConstString(block, result, column_number, column_string, - number_type, left_is_num, input_rows_count); return true; } - /// Comparison between DataTypeEnum and string constant containing the name of an enum element - template - void executeEnumWithConstString( - Block & block, const size_t result, const IColumn * column_number, const ColumnConst * column_string, - const IDataType * type_untyped, const bool left_is_num, size_t input_rows_count) - { - const auto type = static_cast(type_untyped); - - const Field x = castToNearestFieldType(type->getValue(column_string->getValue())); - const auto enum_col = type->createColumnConst(input_rows_count, x); - - executeNumLeftType(block, result, - left_is_num ? column_number : enum_col.get(), - left_is_num ? enum_col.get() : column_number); - } - void executeTuple(Block & block, size_t result, const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1, size_t input_rows_count) { @@ -1124,17 +1080,11 @@ public: bool has_date = left.isDate() || right.isDate(); if (!((both_represented_by_number && !has_date) /// Do not allow compare date and number. - || (left.isStringOrFixedString() && right.isStringOrFixedString()) + || (left.isStringOrFixedString() || right.isStringOrFixedString()) /// Everything can be compared with string by conversion. /// You can compare the date, datetime, or datatime64 and an enumeration with a constant string. - || (left.isString() && right.isDateOrDateTime()) - || (left.isDateOrDateTime() && right.isString()) || (left.isDateOrDateTime() && right.isDateOrDateTime() && left.idx == right.idx) /// only date vs date, or datetime vs datetime || (left.isUUID() && right.isUUID()) - || (left.isUUID() && right.isString()) - || (left.isString() && right.isUUID()) || (left.isEnum() && right.isEnum() && arguments[0]->getName() == arguments[1]->getName()) /// only equivalent enum type values can be compared against - || (left.isEnum() && right.isString()) - || (left.isString() && right.isEnum()) || (left_tuple && right_tuple && left_tuple->getElements().size() == right_tuple->getElements().size()) || (arguments[0]->equals(*arguments[1])))) { @@ -1151,7 +1101,8 @@ public: if (left_tuple && right_tuple) { - auto adaptor = FunctionOverloadResolverAdaptor(std::make_unique(FunctionComparison::create(context))); + auto adaptor = FunctionOverloadResolverAdaptor(std::make_unique( + FunctionComparison::create(context))); size_t size = left_tuple->getElements().size(); for (size_t i = 0; i < size; ++i) @@ -1201,6 +1152,9 @@ public: const bool left_is_num = col_left_untyped->isNumeric(); const bool right_is_num = col_right_untyped->isNumeric(); + const bool left_is_string = isStringOrFixedString(which_left); + const bool right_is_string = isStringOrFixedString(which_right); + bool date_and_datetime = (left_type != right_type) && which_left.isDateOrDateTime() && which_right.isDateOrDateTime(); @@ -1226,64 +1180,14 @@ public: { executeTuple(block, result, col_with_type_and_name_left, col_with_type_and_name_right, input_rows_count); } - else if (which_left.idx != which_right.idx - && (which_left.isDateTime64() || which_right.isDateTime64()) - && (which_left.isStringOrFixedString() || which_right.isStringOrFixedString())) + else if (left_is_string && right_is_string && executeString(block, result, col_left_untyped, col_right_untyped)) + { + } + else if (executeWithConstString( + block, result, col_left_untyped, col_right_untyped, + left_type, right_type, + input_rows_count)) { - /** Special case of comparing DateTime64 against a string. - * - * Can't be moved to executeDateOrDateTimeOrEnumOrUUIDWithConstString() - * since DateTime64 is basically a Decimal, but we do similar things, except type inference. - * Outline: - * - Extract string content - * - Parse it as a ColumnDateTime64 value (same type as DateTime64, means same precision) - * - Fabricate a column with type and name - * - Compare left and right comlumns as DateTime64 columns. - */ - - const size_t datetime64_col_index = which_left.isDateTime64() ? 0 : 1; - const size_t string_col_index = which_left.isStringOrFixedString() ? 0 : 1; - - const auto & datetime64_col_with_type_and_name = block.getByPosition(arguments[datetime64_col_index]); - const auto & string_col_with_type_and_name = block.getByPosition(arguments[string_col_index]); - - if (!isColumnConst(*string_col_with_type_and_name.column)) - throw Exception(getName() + ", illegal column type of argument #" + std::to_string(string_col_index) - + " '" + string_col_with_type_and_name.name + "'" - " expected const String or const FixedString," - " got " + string_col_with_type_and_name.type->getName(), - ErrorCodes::ILLEGAL_COLUMN); - - if (datetime64_col_with_type_and_name.column->size() == 0 || string_col_with_type_and_name.column->size() == 0) - { - // For some reason, when both left and right columns are empty (dry run while building a header block) - // executeDecimal() fills result column with bogus value. - block.getByPosition(result).column = ColumnUInt8::create(); - return; - } - - auto parsed_tmp_column_holder = datetime64_col_with_type_and_name.type->createColumn(); - - { - const StringRef string_value = string_col_with_type_and_name.column->getDataAt(0); - ReadBufferFromMemory in(string_value.data, string_value.size); - datetime64_col_with_type_and_name.type->deserializeAsWholeText(*parsed_tmp_column_holder, in, FormatSettings{}); - - if (!in.eof()) - throw Exception(getName() + ": String is too long for " + datetime64_col_with_type_and_name.type->getName() + " : " + string_value.toString(), ErrorCodes::TOO_LARGE_STRING_SIZE); - } - - // It is necessary to wrap tmp column in ColumnConst to avoid overflow when comparing. - // (non-const columns are expected to have same number of rows as every other column in block). - const ColumnWithTypeAndName parsed_tmp_col_with_type_and_name{ - ColumnConst::create(std::move(parsed_tmp_column_holder), 1), - datetime64_col_with_type_and_name.type, - string_col_with_type_and_name.name}; - - executeDecimal(block, result, - which_left.isDateTime64() ? datetime64_col_with_type_and_name : parsed_tmp_col_with_type_and_name, - which_right.isDateTime64() ? datetime64_col_with_type_and_name : parsed_tmp_col_with_type_and_name); - } else if (isColumnedAsDecimal(left_type) || isColumnedAsDecimal(right_type)) { @@ -1294,19 +1198,10 @@ public: executeDecimal(block, result, col_with_type_and_name_left, col_with_type_and_name_right); } - else if (!left_is_num && !right_is_num && executeString(block, result, col_left_untyped, col_right_untyped)) - { - } else if (left_type->equals(*right_type)) { executeGenericIdenticalTypes(block, result, col_left_untyped, col_right_untyped); } - else if (executeDateOrDateTimeOrEnumOrUUIDWithConstString( - block, result, col_left_untyped, col_right_untyped, - left_type, right_type, - left_is_num, input_rows_count)) - { - } else { executeGeneric(block, result, col_with_type_and_name_left, col_with_type_and_name_right); From f2677a784132adf7b11a2f50e26494a807f0df3c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 01:30:17 +0300 Subject: [PATCH 150/329] Added a test just in case --- tests/queries/0_stateless/01310_enum_comparison.reference | 2 ++ tests/queries/0_stateless/01310_enum_comparison.sql | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/01310_enum_comparison.reference create mode 100644 tests/queries/0_stateless/01310_enum_comparison.sql diff --git a/tests/queries/0_stateless/01310_enum_comparison.reference b/tests/queries/0_stateless/01310_enum_comparison.reference new file mode 100644 index 00000000000..b261da18d51 --- /dev/null +++ b/tests/queries/0_stateless/01310_enum_comparison.reference @@ -0,0 +1,2 @@ +1 +0 diff --git a/tests/queries/0_stateless/01310_enum_comparison.sql b/tests/queries/0_stateless/01310_enum_comparison.sql new file mode 100644 index 00000000000..26901a61b2b --- /dev/null +++ b/tests/queries/0_stateless/01310_enum_comparison.sql @@ -0,0 +1,6 @@ +CREATE TEMPORARY TABLE enum (x Enum('hello' = 1, 'world' = 2)); +INSERT INTO enum VALUES ('hello'); + +SELECT count() FROM enum WHERE x = 'hello'; +SELECT count() FROM enum WHERE x = 'world'; +SELECT count() FROM enum WHERE x = 'xyz'; -- { serverError 36 } From 41afea0165aa2bf66fd46903044bb08c874be54e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 02:10:10 +0300 Subject: [PATCH 151/329] Fix style --- src/Functions/FunctionsComparison.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 12da4c772d1..9cd13df826d 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -52,7 +52,6 @@ namespace DB namespace ErrorCodes { - extern const int TOO_LARGE_STRING_SIZE; extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int LOGICAL_ERROR; From 713f8f0b2246be3381f2e7d1967b10173e24523c Mon Sep 17 00:00:00 2001 From: Volodymyr Kuznetsov Date: Sat, 13 Jun 2020 17:21:33 -0700 Subject: [PATCH 152/329] Added groupArrayArray and groupUniqArrayArray to SimpleAggregateFunction --- .../DataTypeCustomSimpleAggregateFunction.cpp | 2 +- .../00915_simple_aggregate_function.reference | 6 +++--- .../00915_simple_aggregate_function.sql | 14 ++++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp index 2ddce184cce..8b31a93dfe9 100644 --- a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp +++ b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp @@ -30,7 +30,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -static const std::vector supported_functions{"any", "anyLast", "min", "max", "sum", "groupBitAnd", "groupBitOr", "groupBitXor", "sumMap"}; +static const std::vector supported_functions{"any", "anyLast", "min", "max", "sum", "groupBitAnd", "groupBitOr", "groupBitXor", "sumMap", "groupArrayArray", "groupUniqArrayArray"}; String DataTypeCustomSimpleAggregateFunction::getName() const diff --git a/tests/queries/0_stateless/00915_simple_aggregate_function.reference b/tests/queries/0_stateless/00915_simple_aggregate_function.reference index d9e0a92cb01..771c19f2227 100644 --- a/tests/queries/0_stateless/00915_simple_aggregate_function.reference +++ b/tests/queries/0_stateless/00915_simple_aggregate_function.reference @@ -39,6 +39,6 @@ SimpleAggregateFunction(sum, Float64) 7 14 8 16 9 18 -1 1 2 2.2.2.2 3 ([1,2,3],[2,1,1]) -10 2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222 20 20.20.20.20 5 ([2,3,4],[2,1,1]) -SimpleAggregateFunction(anyLast, Nullable(String)) SimpleAggregateFunction(anyLast, LowCardinality(Nullable(String))) SimpleAggregateFunction(anyLast, IPv4) SimpleAggregateFunction(groupBitOr, UInt32) SimpleAggregateFunction(sumMap, Tuple(Array(Int32), Array(Int64))) +1 1 2 2.2.2.2 3 ([1,2,3],[2,1,1]) [1,2,2,3,4] [4,2,1,3] +10 2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222 20 20.20.20.20 5 ([2,3,4],[2,1,1]) [] [] +SimpleAggregateFunction(anyLast, Nullable(String)) SimpleAggregateFunction(anyLast, LowCardinality(Nullable(String))) SimpleAggregateFunction(anyLast, IPv4) SimpleAggregateFunction(groupBitOr, UInt32) SimpleAggregateFunction(sumMap, Tuple(Array(Int32), Array(Int64))) SimpleAggregateFunction(groupArrayArray, Array(Int32)) SimpleAggregateFunction(groupUniqArrayArray, Array(Int32)) diff --git a/tests/queries/0_stateless/00915_simple_aggregate_function.sql b/tests/queries/0_stateless/00915_simple_aggregate_function.sql index ba4935a6518..8cf0e032702 100644 --- a/tests/queries/0_stateless/00915_simple_aggregate_function.sql +++ b/tests/queries/0_stateless/00915_simple_aggregate_function.sql @@ -25,16 +25,18 @@ create table simple ( low_str SimpleAggregateFunction(anyLast,LowCardinality(Nullable(String))), ip SimpleAggregateFunction(anyLast,IPv4), status SimpleAggregateFunction(groupBitOr, UInt32), - tup SimpleAggregateFunction(sumMap, Tuple(Array(Int32), Array(Int64))) + tup SimpleAggregateFunction(sumMap, Tuple(Array(Int32), Array(Int64))), + arr SimpleAggregateFunction(groupArrayArray, Array(Int32)), + uniq_arr SimpleAggregateFunction(groupUniqArrayArray, Array(Int32)) ) engine=AggregatingMergeTree order by id; -insert into simple values(1,'1','1','1.1.1.1', 1, ([1,2], [1,1])); -insert into simple values(1,null,'2','2.2.2.2', 2, ([1,3], [1,1])); +insert into simple values(1,'1','1','1.1.1.1', 1, ([1,2], [1,1]), [1,2], [1,2]); +insert into simple values(1,null,'2','2.2.2.2', 2, ([1,3], [1,1]), [2,3,4], [2,3,4]); -- String longer then MAX_SMALL_STRING_SIZE (actual string length is 100) -insert into simple values(10,'10','10','10.10.10.10', 4, ([2,3], [1,1])); -insert into simple values(10,'2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222','20','20.20.20.20', 1, ([2, 4], [1,1])); +insert into simple values(10,'10','10','10.10.10.10', 4, ([2,3], [1,1]), [], []); +insert into simple values(10,'2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222','20','20.20.20.20', 1, ([2, 4], [1,1]), [], []); select * from simple final order by id; -select toTypeName(nullable_str),toTypeName(low_str),toTypeName(ip),toTypeName(status), toTypeName(tup) from simple limit 1; +select toTypeName(nullable_str),toTypeName(low_str),toTypeName(ip),toTypeName(status), toTypeName(tup), toTypeName(arr), toTypeName(uniq_arr) from simple limit 1; optimize table simple final; From 30f1f8811855a75d282eeba5cf2fd23191e2b656 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 03:43:42 +0300 Subject: [PATCH 153/329] Allow case-insensitive regexps; added a test #11101 --- src/Common/OptimizedRegularExpression.cpp | 37 ++++++++++++++++++- .../01312_case_insensitive_regexp.reference | 8 ++++ .../01312_case_insensitive_regexp.sql | 8 ++++ 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01312_case_insensitive_regexp.reference create mode 100644 tests/queries/0_stateless/01312_case_insensitive_regexp.sql diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index 8c4aa00f171..1464923e6ab 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -38,6 +38,7 @@ void OptimizedRegularExpressionImpl::analyze( required_substring_is_prefix = false; required_substring.clear(); bool has_alternative_on_depth_0 = false; + bool has_case_insensitive_flag = false; /// Substring with a position. using Substring = std::pair; @@ -65,7 +66,17 @@ void OptimizedRegularExpressionImpl::analyze( switch (*pos) { - case '|': case '(': case ')': case '^': case '$': case '.': case '[': case '?': case '*': case '+': case '{': + case '|': + case '(': + case ')': + case '^': + case '$': + case '.': + case '[': + case '?': + case '*': + case '+': + case '{': if (depth == 0 && !in_curly_braces && !in_square_braces) { if (last_substring->first.empty()) @@ -110,6 +121,28 @@ void OptimizedRegularExpressionImpl::analyze( trivial_substrings.resize(trivial_substrings.size() + 1); last_substring = &trivial_substrings.back(); } + + /// Check for case-insensitive flag. + if (pos + 1 < end && pos[1] == '?') + { + for (size_t offset = 2; pos + offset < end; ++offset) + { + if (pos[offset] == '-' /// it means flag negation + /// various possible flags, actually only imsU are supported by re2 + || (pos[offset] >= 'a' && pos[offset] <= 'z') + || (pos[offset] >= 'A' && pos[offset] <= 'Z')) + { + if (pos[offset] == 'i') + { + /// Actually it can be negated case-insensitive flag. But we don't care. + has_case_insensitive_flag = true; + break; + } + } + else + break; + } + } } ++pos; break; @@ -209,7 +242,7 @@ void OptimizedRegularExpressionImpl::analyze( if (!is_trivial) { - if (!has_alternative_on_depth_0) + if (!has_alternative_on_depth_0 && !has_case_insensitive_flag) { /// We choose the non-alternative substring of the maximum length for first search. diff --git a/tests/queries/0_stateless/01312_case_insensitive_regexp.reference b/tests/queries/0_stateless/01312_case_insensitive_regexp.reference new file mode 100644 index 00000000000..c18b4e9b082 --- /dev/null +++ b/tests/queries/0_stateless/01312_case_insensitive_regexp.reference @@ -0,0 +1,8 @@ +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01312_case_insensitive_regexp.sql b/tests/queries/0_stateless/01312_case_insensitive_regexp.sql new file mode 100644 index 00000000000..ca13989599d --- /dev/null +++ b/tests/queries/0_stateless/01312_case_insensitive_regexp.sql @@ -0,0 +1,8 @@ +SELECT match('Too late', 'Too late'); +select match('Too late', '(?i)Too late'); +select match('Too late', '(?i)too late'); +select match('Too late', '(?i:too late)'); +select match('Too late', '(?i)to{2} late'); +select match('Too late', '(?i)to(?)o late'); +select match('Too late', '(?i)to+ late'); +select match('Too late', '(?i)to(?:o|o) late'); From 970a8e3ecc7fbabb2590fd8a0b093b472e6adfb6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 03:56:13 +0300 Subject: [PATCH 154/329] Skip empty URL parameters #10749 --- src/Server/HTTPHandler.cpp | 4 ++++ .../0_stateless/01312_skip_empty_params.reference | 3 +++ tests/queries/0_stateless/01312_skip_empty_params.sh | 10 ++++++++++ 3 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/01312_skip_empty_params.reference create mode 100755 tests/queries/0_stateless/01312_skip_empty_params.sh diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 84d23f10a55..e866af2f49b 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -430,6 +430,10 @@ void HTTPHandler::processQuery( auto param_could_be_skipped = [&] (const String & name) { + /// Empty parameter appears when URL like ?&a=b or a=b&&c=d. Just skip them for user's convenience. + if (name.empty()) + return true; + if (reserved_param_names.count(name)) return true; diff --git a/tests/queries/0_stateless/01312_skip_empty_params.reference b/tests/queries/0_stateless/01312_skip_empty_params.reference new file mode 100644 index 00000000000..e8183f05f5d --- /dev/null +++ b/tests/queries/0_stateless/01312_skip_empty_params.reference @@ -0,0 +1,3 @@ +1 +1 +1 diff --git a/tests/queries/0_stateless/01312_skip_empty_params.sh b/tests/queries/0_stateless/01312_skip_empty_params.sh new file mode 100755 index 00000000000..2e3541aee35 --- /dev/null +++ b/tests/queries/0_stateless/01312_skip_empty_params.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&query=select%201&log_queries=1" +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&&query=select%201&log_queries=1" +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&query=select%201&&&log_queries=1" From 6c278fee616e8326b40b2dbf62bed98be77a5ba6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 04:07:47 +0300 Subject: [PATCH 155/329] Better exception message --- src/Core/SettingsCollectionImpl.h | 13 ++++++++++++- src/Interpreters/Context.cpp | 11 ++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/Core/SettingsCollectionImpl.h b/src/Core/SettingsCollectionImpl.h index 8210b04e2da..877567a7caf 100644 --- a/src/Core/SettingsCollectionImpl.h +++ b/src/Core/SettingsCollectionImpl.h @@ -7,6 +7,8 @@ */ #include +#include + namespace DB { @@ -91,7 +93,16 @@ Field SettingsCollection::const_reference::getValue() const template Field SettingsCollection::valueToCorrespondingType(size_t index, const Field & value) { - return members()[index].value_to_corresponding_type(value); + try + { + return members()[index].value_to_corresponding_type(value); + } + catch (Exception & e) + { + e.addMessage(fmt::format("in attempt to set the value of setting to {}", + applyVisitor(FieldVisitorToString(), value))); + throw; + } } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 1431f3fd62c..cb780443e03 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -982,7 +982,16 @@ void Context::setSetting(const StringRef & name, const Field & value) void Context::applySettingChange(const SettingChange & change) { - setSetting(change.name, change.value); + try + { + setSetting(change.name, change.value); + } + catch (Exception & e) + { + e.addMessage(fmt::format("in attempt to set the value of setting '{}' to {}", + change.name, applyVisitor(FieldVisitorToString(), change.value))); + throw; + } } From 400e9fb64f68e020907f9b4a9251b72c415455d4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 04:23:53 +0300 Subject: [PATCH 156/329] Allow to drop table if there is no metadata in ZooKeeper; allow to rename --- src/Storages/StorageReplicatedMergeTree.cpp | 22 +++++++++++++++++++-- src/Storages/StorageReplicatedMergeTree.h | 2 ++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 57535466558..5931bca17ea 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -244,6 +244,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( { LOG_WARNING(log, "No metadata in ZooKeeper: table will be in readonly mode."); is_readonly = true; + has_metadata_in_zookeeper = false; return; } @@ -620,9 +621,14 @@ void StorageReplicatedMergeTree::createReplica() void StorageReplicatedMergeTree::drop() { + /// There is also the case when user has configured ClickHouse to wrong ZooKeeper cluster, + /// in this case, has_metadata_in_zookeeper = false, and we also permit to drop the table. + + if (has_metadata_in_zookeeper) { auto zookeeper = tryGetZooKeeper(); + /// If probably there is metadata in ZooKeeper, we don't allow to drop the table. if (is_readonly || !zookeeper) throw Exception("Can't drop readonly replicated table (need to drop data in ZooKeeper as well)", ErrorCodes::TABLE_IS_READ_ONLY); @@ -4032,8 +4038,20 @@ void StorageReplicatedMergeTree::rename(const String & new_path_to_table_data, c MergeTreeData::rename(new_path_to_table_data, new_table_id); /// Update table name in zookeeper - auto zookeeper = getZooKeeper(); - zookeeper->set(replica_path + "/host", getReplicatedMergeTreeAddress().toString()); + if (!is_readonly) + { + /// We don't do it for readonly tables, because it will be updated on next table startup. + /// It is also Ok to skip ZK error for the same reason. + try + { + auto zookeeper = getZooKeeper(); + zookeeper->set(replica_path + "/host", getReplicatedMergeTreeAddress().toString()); + } + catch (Coordination::Exception & e) + { + LOG_WARNING(log, "Cannot update the value of 'host' node (replica address) in ZooKeeper: {}", e.displayText()); + } + } /// TODO: You can update names of loggers. } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index ec38eb7e842..52ce1aada08 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -210,6 +210,8 @@ private: /// If true, the table is offline and can not be written to it. std::atomic_bool is_readonly {false}; + /// If false - ZooKeeper is available, but there is no table metadata. It's safe to drop table in this case. + bool has_metadata_in_zookeeper = true; String zookeeper_path; String replica_name; From e1317ef8ae6acc7ac40c3b6985bbd97828880dd2 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sun, 14 Jun 2020 09:44:05 +0800 Subject: [PATCH 157/329] ISSUES-7572 fix test failure --- tests/integration/test_http_handlers_config/test.py | 6 +++--- .../test_prometheus_handler/config.xml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index c18c22acbb2..b31913ba962 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -119,10 +119,10 @@ def test_defaults_http_handlers(): assert 'Default server response' == cluster.instance.http_request('', method='GET').content assert 200 == cluster.instance.http_request('ping', method='GET').status_code - assert 'Ok\n' == cluster.instance.http_request('ping', method='GET').content + assert 'Ok.\n' == cluster.instance.http_request('ping', method='GET').content assert 200 == cluster.instance.http_request('replicas_status', method='GET').status_code - assert 'Ok\n' == cluster.instance.http_request('replicas_status', method='GET').content + assert 'Ok.\n' == cluster.instance.http_request('replicas_status', method='GET').content def test_prometheus_handler(): with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster: @@ -144,4 +144,4 @@ def test_replicas_status_handler(): assert 404 == cluster.instance.http_request('test_replicas_status', method='POST', headers={'XXX': 'xxx'}).status_code assert 200 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).status_code - assert 'Ok\n' == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).content + assert 'Ok.\n' == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).content diff --git a/tests/integration/test_http_handlers_config/test_prometheus_handler/config.xml b/tests/integration/test_http_handlers_config/test_prometheus_handler/config.xml index 7c80649cee2..8ace97a66dc 100644 --- a/tests/integration/test_http_handlers_config/test_prometheus_handler/config.xml +++ b/tests/integration/test_http_handlers_config/test_prometheus_handler/config.xml @@ -7,7 +7,7 @@ xxx /test_prometheus - replicas_status + prometheus true true true From fae12d5e42048805873ee8c2fb6cd6eee8ae1e3b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 04:45:03 +0300 Subject: [PATCH 158/329] Return NULL or zero when value is not parsed completely in parseDateTimeBestEffortOr* functions --- src/Functions/FunctionsConversion.h | 4 ++-- ...3_parse_date_time_best_effort_null_zero.reference | 6 ++++++ .../01313_parse_date_time_best_effort_null_zero.sql | 12 ++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.reference create mode 100644 tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.sql diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 879b885cf66..83417a3229b 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -722,10 +722,10 @@ struct ConvertThroughParsing parsed = ToDataType::tryReadText(vec_to[i], read_buffer, ToDataType::maxPrecision(), vec_to.getScale()); else parsed = tryParseImpl(vec_to[i], read_buffer, local_time_zone); - - parsed = parsed && isAllRead(read_buffer); } + parsed = parsed && isAllRead(read_buffer); + if (!parsed) vec_to[i] = 0; diff --git a/tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.reference b/tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.reference new file mode 100644 index 00000000000..90bb776ca10 --- /dev/null +++ b/tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.reference @@ -0,0 +1,6 @@ +\N +0000-00-00 00:00:00 +\N +0000-00-00 00:00:00.000 +\N +0000-00-00 00:00:00 diff --git a/tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.sql b/tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.sql new file mode 100644 index 00000000000..69b66b46df7 --- /dev/null +++ b/tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.sql @@ -0,0 +1,12 @@ +SELECT parseDateTimeBestEffort(''); -- { serverError 6 } +SELECT parseDateTimeBestEffortOrNull(''); +SELECT parseDateTimeBestEffortOrZero(''); + +SELECT parseDateTime64BestEffort(''); -- { serverError 6 } +SELECT parseDateTime64BestEffortOrNull(''); +SELECT parseDateTime64BestEffortOrZero(''); + +SET date_time_input_format = 'best_effort'; +SELECT toDateTime(''); -- { serverError 41 } +SELECT toDateTimeOrNull(''); +SELECT toDateTimeOrZero(''); From d6cf62e5872036099bb607bd075e68275aadf642 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 04:56:22 +0300 Subject: [PATCH 159/329] Added column "position" to system.columns and "column_position" to system.parts_columns --- src/Storages/System/StorageSystemColumns.cpp | 5 +++++ src/Storages/System/StorageSystemPartsColumns.cpp | 5 ++++- .../01314_position_in_system_columns.reference | 6 ++++++ .../0_stateless/01314_position_in_system_columns.sql | 8 ++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01314_position_in_system_columns.reference create mode 100644 tests/queries/0_stateless/01314_position_in_system_columns.sql diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index b4f5da22c17..90e52ad373e 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -32,6 +32,7 @@ StorageSystemColumns::StorageSystemColumns(const std::string & name_) { "table", std::make_shared() }, { "name", std::make_shared() }, { "type", std::make_shared() }, + { "position", std::make_shared() }, { "default_kind", std::make_shared() }, { "default_expression", std::make_shared() }, { "data_compressed_bytes", std::make_shared() }, @@ -131,8 +132,10 @@ protected: bool check_access_for_columns = check_access_for_tables && !access->isGranted(AccessType::SHOW_COLUMNS, database_name, table_name); + size_t position = 0; for (const auto & column : columns) { + ++position; if (check_access_for_columns && !access->isGranted(AccessType::SHOW_COLUMNS, database_name, table_name, column.name)) continue; @@ -147,6 +150,8 @@ protected: res_columns[res_index++]->insert(column.name); if (columns_mask[src_index++]) res_columns[res_index++]->insert(column.type->getName()); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(position); if (column.default_desc.expression) { diff --git a/src/Storages/System/StorageSystemPartsColumns.cpp b/src/Storages/System/StorageSystemPartsColumns.cpp index b8acdc5f995..479621fd47f 100644 --- a/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/src/Storages/System/StorageSystemPartsColumns.cpp @@ -49,6 +49,7 @@ StorageSystemPartsColumns::StorageSystemPartsColumns(const std::string & name_) {"column", std::make_shared()}, {"type", std::make_shared()}, + {"column_position", std::make_shared()}, {"default_kind", std::make_shared()}, {"default_expression", std::make_shared()}, {"column_bytes_on_disk", std::make_shared()}, @@ -101,9 +102,10 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns_, co using State = IMergeTreeDataPart::State; + size_t column_position = 0; for (const auto & column : part->getColumns()) - { + ++column_position; size_t j = 0; { WriteBufferFromOwnString out; @@ -143,6 +145,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns_, co columns_[j++]->insert(column.name); columns_[j++]->insert(column.type->getName()); + columns_[j++]->insert(column_position); auto column_info_it = columns_info.find(column.name); if (column_info_it != columns_info.end()) diff --git a/tests/queries/0_stateless/01314_position_in_system_columns.reference b/tests/queries/0_stateless/01314_position_in_system_columns.reference new file mode 100644 index 00000000000..32e0ae5900e --- /dev/null +++ b/tests/queries/0_stateless/01314_position_in_system_columns.reference @@ -0,0 +1,6 @@ +x UInt8 1 +y String 2 +z Array(String) 3 +x UInt8 1 +y String 2 +z Array(String) 3 diff --git a/tests/queries/0_stateless/01314_position_in_system_columns.sql b/tests/queries/0_stateless/01314_position_in_system_columns.sql new file mode 100644 index 00000000000..7bb0f3b5a96 --- /dev/null +++ b/tests/queries/0_stateless/01314_position_in_system_columns.sql @@ -0,0 +1,8 @@ +DROP TABLE IF EXISTS test; +CREATE TABLE test (x UInt8, y String, z Array(String)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO test (x) VALUES (1); + +SELECT name, type, position FROM system.columns WHERE database = currentDatabase() AND table = 'test'; +SELECT column, type, column_position FROM system.parts_columns WHERE database = currentDatabase() AND table = 'test'; + +DROP TABLE test; From 4a052f60c7eb40ee0c0a323a5039f9f0931c3bd3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 06:10:35 +0300 Subject: [PATCH 160/329] Don't use debug info from ELF file if it doesn't correspond to the running binary. --- src/Common/Elf.cpp | 46 ++++++++++++++++++++++++++++++++++++++ src/Common/Elf.h | 7 ++++++ src/Common/SymbolIndex.cpp | 37 ++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) diff --git a/src/Common/Elf.cpp b/src/Common/Elf.cpp index 11d454abd71..ee32586d95e 100644 --- a/src/Common/Elf.cpp +++ b/src/Common/Elf.cpp @@ -54,6 +54,18 @@ Elf::Elf(const std::string & path) throw Exception("The ELF is truncated (section names string table points after end of file)", ErrorCodes::CANNOT_PARSE_ELF); section_names = reinterpret_cast(mapped + section_names_offset); + + /// Get program headers + + ElfOff program_header_offset = header->e_phoff; + uint16_t program_header_num_entries = header->e_phnum; + + if (!program_header_offset + || !program_header_num_entries + || program_header_offset + program_header_num_entries * sizeof(ElfPhdr) > elf_size) + throw Exception("The ELF is truncated (program header points after end of file)", ErrorCodes::CANNOT_PARSE_ELF); + + program_headers = reinterpret_cast(mapped + program_header_offset); } @@ -104,6 +116,40 @@ std::optional Elf::findSectionByName(const char * name) const } +String Elf::getBuildID() const +{ + for (size_t idx = 0; idx < header->e_phnum; ++idx) + { + const ElfPhdr & phdr = program_headers[idx]; + + if (phdr.p_type == PT_NOTE) + return getBuildID(mapped + phdr.p_offset, phdr.p_filesz); + } + return {}; +} + + +String Elf::getBuildID(const char * nhdr_pos, size_t size) +{ + const char * nhdr_end = nhdr_pos + size; + + while (nhdr_pos < nhdr_end) + { + const ElfNhdr & nhdr = *reinterpret_cast(nhdr_pos); + + nhdr_pos += sizeof(ElfNhdr) + nhdr.n_namesz; + if (nhdr.n_type == NT_GNU_BUILD_ID) + { + const char * build_id = nhdr_pos; + return {build_id, nhdr.n_descsz}; + } + nhdr_pos += nhdr.n_descsz; + } + + return {}; +} + + const char * Elf::Section::name() const { if (!elf.section_names) diff --git a/src/Common/Elf.h b/src/Common/Elf.h index f3aafc8e5a9..632d7e6f0b1 100644 --- a/src/Common/Elf.h +++ b/src/Common/Elf.h @@ -17,6 +17,7 @@ using ElfEhdr = ElfW(Ehdr); using ElfOff = ElfW(Off); using ElfPhdr = ElfW(Phdr); using ElfShdr = ElfW(Shdr); +using ElfNhdr = ElfW(Nhdr); using ElfSym = ElfW(Sym); @@ -53,12 +54,18 @@ public: const char * end() const { return mapped + elf_size; } size_t size() const { return elf_size; } + /// Obtain build id from PT_NOTES section of program headers. Return empty string if does not exist. + /// The string is returned in binary. Note that "readelf -n ./clickhouse-server" prints it in hex. + String getBuildID() const; + static String getBuildID(const char * nhdr_pos, size_t size); + private: MMapReadBufferFromFile in; size_t elf_size; const char * mapped; const ElfEhdr * header; const ElfShdr * section_headers; + const ElfPhdr * program_headers; const char * section_names = nullptr; }; diff --git a/src/Common/SymbolIndex.cpp b/src/Common/SymbolIndex.cpp index 482c6fd0bad..54789695dd1 100644 --- a/src/Common/SymbolIndex.cpp +++ b/src/Common/SymbolIndex.cpp @@ -196,6 +196,20 @@ void collectSymbolsFromProgramHeaders(dl_phdr_info * info, } +String getBuildIDFromProgramHeaders(dl_phdr_info * info) +{ + for (size_t header_index = 0; header_index < info->dlpi_phnum; ++header_index) + { + const ElfPhdr & phdr = info->dlpi_phdr[header_index]; + if (phdr.p_type != PT_NOTE) + continue; + + return Elf::getBuildID(reinterpret_cast(info->dlpi_addr + phdr.p_vaddr), phdr.p_memsz); + } + return {}; +} + + void collectSymbolsFromELFSymbolTable( dl_phdr_info * info, const Elf & elf, @@ -283,8 +297,31 @@ void collectSymbolsFromELF(dl_phdr_info * info, object_name = std::filesystem::exists(debug_info_path) ? debug_info_path : canonical_path; + /// But we have to compare Build ID to check that debug info corresponds to the same executable. + String our_build_id = getBuildIDFromProgramHeaders(info); + SymbolIndex::Object object; object.elf = std::make_unique(object_name); + + String file_build_id = object.elf->getBuildID(); + + if (our_build_id != file_build_id) + { + /// If debug info doesn't correspond to our binary, fallback to the info in our binary. + if (object_name != canonical_path) + { + object_name = canonical_path; + object.elf = std::make_unique(object_name); + + /// But it can still be outdated, for example, if executable file was deleted from filesystem and replaced by another file. + file_build_id = object.elf->getBuildID(); + if (our_build_id != file_build_id) + return; + } + else + return; + } + object.address_begin = reinterpret_cast(info->dlpi_addr); object.address_end = reinterpret_cast(info->dlpi_addr + object.elf->size()); object.name = object_name; From cb395ff099ff6f6602356d0ee7ab14690ad5bb80 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 07:04:19 +0300 Subject: [PATCH 161/329] Update test --- tests/queries/0_stateless/00700_decimal_compare.reference | 2 ++ tests/queries/0_stateless/00700_decimal_compare.sql | 2 +- tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql | 8 ++++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/00700_decimal_compare.reference b/tests/queries/0_stateless/00700_decimal_compare.reference index 32f0b0a6dea..2325847045f 100644 --- a/tests/queries/0_stateless/00700_decimal_compare.reference +++ b/tests/queries/0_stateless/00700_decimal_compare.reference @@ -1,3 +1,5 @@ +0 +1 -42 -42 1 0 0 0 1 1 42 42 1 0 0 0 1 1 -42 -42.42000 0 0 1 1 0 1 diff --git a/tests/queries/0_stateless/00700_decimal_compare.sql b/tests/queries/0_stateless/00700_decimal_compare.sql index 24b4ce588e5..ae2f5790570 100644 --- a/tests/queries/0_stateless/00700_decimal_compare.sql +++ b/tests/queries/0_stateless/00700_decimal_compare.sql @@ -19,7 +19,7 @@ INSERT INTO decimal (a, b, c, d, e, f, g, h, i, j) VALUES (-42, -42, -42, -0.42, SELECT a > toFloat64(0) FROM decimal; -- { serverError 43 } SELECT g > toFloat32(0) FROM decimal; -- { serverError 43 } -SELECT a > '0.0' FROM decimal; -- { serverError 43 } +SELECT a > '0.0' FROM decimal ORDER BY a; SELECT a, b, a = b, a < b, a > b, a != b, a <= b, a >= b FROM decimal ORDER BY a; SELECT a, g, a = g, a < g, a > g, a != g, a <= g, a >= g FROM decimal ORDER BY a; diff --git a/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql b/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql index c65bf668d71..7848b4aaf24 100644 --- a/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql +++ b/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql @@ -1,9 +1,9 @@ -- Error cases: -- non-const string column -WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT DT64 = materialize(S); -- {serverError 44} -WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT materialize(S) = toDateTime64(S, 3); -- {serverError 44} -WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT * WHERE DT64 = materialize(S); -- {serverError 44} -WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT * WHERE materialize(S) = DT64; -- {serverError 44} +WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT DT64 = materialize(S); -- {serverError 43} +WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT materialize(S) = toDateTime64(S, 3); -- {serverError 43} +WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT * WHERE DT64 = materialize(S); -- {serverError 43} +WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT * WHERE materialize(S) = DT64; -- {serverError 43} SELECT * WHERE toDateTime64(123.345, 3) == 'ABCD'; -- {serverError 131} -- invalid DateTime64 string SELECT * WHERE toDateTime64(123.345, 3) == '2020-02-05 14:34:12.33333333333333333333333333333333333333333333333333333333'; -- {serverError 131} -- invalid string length From 59d4df19f03ab10588f043efa4f24e3292c2698d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 07:05:31 +0300 Subject: [PATCH 162/329] Update test --- tests/queries/0_stateless/00981_no_virtual_columns.reference | 2 +- tests/queries/0_stateless/00981_no_virtual_columns.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00981_no_virtual_columns.reference b/tests/queries/0_stateless/00981_no_virtual_columns.reference index a7ec77dc030..587be6b4c3f 100644 --- a/tests/queries/0_stateless/00981_no_virtual_columns.reference +++ b/tests/queries/0_stateless/00981_no_virtual_columns.reference @@ -1 +1 @@ -default merge_ab x UInt8 0 0 0 0 0 0 0 +x diff --git a/tests/queries/0_stateless/00981_no_virtual_columns.sql b/tests/queries/0_stateless/00981_no_virtual_columns.sql index 476377b4ddf..b3946154581 100644 --- a/tests/queries/0_stateless/00981_no_virtual_columns.sql +++ b/tests/queries/0_stateless/00981_no_virtual_columns.sql @@ -6,7 +6,7 @@ CREATE TABLE merge_a (x UInt8) ENGINE = StripeLog; CREATE TABLE merge_b (x UInt8) ENGINE = StripeLog; CREATE TABLE merge_ab AS merge(currentDatabase(), '^merge_[ab]$'); -SELECT * FROM system.columns WHERE database = currentDatabase() AND table = 'merge_ab'; +SELECT name FROM system.columns WHERE database = currentDatabase() AND table = 'merge_ab'; DROP TABLE merge_a; DROP TABLE merge_b; From fb040ef09fea5b59d573ae874192f62b07f0d5b7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 07:09:02 +0300 Subject: [PATCH 163/329] Update test result (now it is better) --- .../00569_parse_date_time_best_effort.reference | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference b/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference index b873e8b848d..bf11e1c1d71 100644 --- a/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference +++ b/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference @@ -13,7 +13,7 @@ 11 Feb 2018 06:40:50 +0300 2018-02-11 03:40:50 2018-02-11 03:40:50 17 Apr 2 1:2:3 2000-04-17 01:02:03 2000-04-17 01:02:03 19700102 01:00:00 1970-01-02 01:00:00 1970-01-02 01:00:00 - 1970010201:00:00 2032-06-06 02:03:21 2032-06-06 02:03:21 + 1970010201:00:00 ᴺᵁᴸᴸ 0000-00-00 00:00:00 19700102010203 1970-01-02 01:02:03 1970-01-02 01:02:03 19700102010203Z 1970-01-02 01:02:03 1970-01-02 01:02:03 1970/01/02 010203Z 1970-01-02 01:02:03 1970-01-02 01:02:03 @@ -61,7 +61,7 @@ 2017/01/32 0000-00-00 00:00:00 0000-00-00 00:00:00 2017-01 MSD Jun 2017-05-31 20:00:00 2017-05-31 20:00:00 201701 MSD Jun 2017-05-31 20:00:00 2017-05-31 20:00:00 - 2017 25 1:2:3 0000-00-00 00:00:00 0000-00-00 00:00:00 + 2017 25 1:2:3 ᴺᵁᴸᴸ 0000-00-00 00:00:00 2017 25 Apr 1:2:3 2017-04-01 01:02:03 2017-04-01 01:02:03 2017 Apr 01 11:22:33 2017-04-01 11:22:33 2017-04-01 11:22:33 2017 Apr 02 01/02/03 UTC+0300 ᴺᵁᴸᴸ 0000-00-00 00:00:00 @@ -74,8 +74,8 @@ 2017 Apr 02 1:2:3 2017-04-02 01:02:03 2017-04-02 01:02:03 2017 Apr 02 1:2:33 2017-04-02 01:02:33 2017-04-02 01:02:33 2017 Apr 02 1:2:3 MSK 2017-04-01 22:02:03 2017-04-01 22:02:03 - 2017 Apr 02 1:2:3 MSK 2017 2017-04-01 22:02:03 2017-04-01 22:02:03 - 2017 Apr 02 1:2:3 MSK 2018 2017-04-01 22:02:03 2017-04-01 22:02:03 + 2017 Apr 02 1:2:3 MSK 2017 ᴺᵁᴸᴸ 0000-00-00 00:00:00 + 2017 Apr 02 1:2:3 MSK 2018 ᴺᵁᴸᴸ 0000-00-00 00:00:00 2017 Apr 02 1:2:3 UTC+0000 2017-04-02 01:02:03 2017-04-02 01:02:03 2017 Apr 02 1:2:3 UTC+0300 2017-04-01 22:02:03 2017-04-01 22:02:03 2017 Apr 02 1:2:3 UTC+0400 2017-04-01 21:02:03 2017-04-01 21:02:03 @@ -101,6 +101,6 @@ 25 Jan 2017 1:2:3 Z PM 2017-01-25 13:02:03 2017-01-25 13:02:03 25 Jan 2017 1:2:3Z PM 2017-01-25 13:02:03 2017-01-25 13:02:03 25 Jan 2017 1:2:3 Z PM +03:00 2017-01-25 10:02:03 2017-01-25 10:02:03 - Jun, 11 Feb 2018 06:40:50 +0300 2000-06-01 00:00:00 2000-06-01 00:00:00 + Jun, 11 Feb 2018 06:40:50 +0300 ᴺᵁᴸᴸ 0000-00-00 00:00:00 Sun 11 Feb 2018 06:40:50 +0300 2018-02-11 03:40:50 2018-02-11 03:40:50 Sun, 11 Feb 2018 06:40:50 +0300 2018-02-11 03:40:50 2018-02-11 03:40:50 From e24576c56c4d12bf5ab66500c4462c9163c85976 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 07:13:27 +0300 Subject: [PATCH 164/329] Update test --- .../0_stateless/00578_merge_table_shadow_virtual_column.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00578_merge_table_shadow_virtual_column.sql b/tests/queries/0_stateless/00578_merge_table_shadow_virtual_column.sql index 3071e307517..e729bfdf188 100644 --- a/tests/queries/0_stateless/00578_merge_table_shadow_virtual_column.sql +++ b/tests/queries/0_stateless/00578_merge_table_shadow_virtual_column.sql @@ -4,7 +4,7 @@ DROP TABLE IF EXISTS numbers2; CREATE TABLE numbers1 ENGINE = Memory AS SELECT number as _table FROM numbers(1000); CREATE TABLE numbers2 ENGINE = Memory AS SELECT number as _table FROM numbers(1000); -SELECT count() FROM merge(currentDatabase(), '^numbers\\d+$') WHERE _table='numbers1'; -- { serverError 43 } +SELECT count() FROM merge(currentDatabase(), '^numbers\\d+$') WHERE _table='numbers1'; -- { serverError 53 } SELECT count() FROM merge(currentDatabase(), '^numbers\\d+$') WHERE _table=1; DROP TABLE numbers1; From eec5abde071d98942446f3292b4a50ed0284cb4e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 07:16:23 +0300 Subject: [PATCH 165/329] Fix test --- src/Interpreters/convertFieldToType.cpp | 65 ------------------------- 1 file changed, 65 deletions(-) diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 3cb774596c0..6328ed76924 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -124,42 +124,6 @@ static Field convertDecimalType(const Field & from, const To & type) } -DayNum stringToDate(const String & s) -{ - ReadBufferFromString in(s); - DayNum date{}; - - readDateText(date, in); - if (!in.eof()) - throw Exception("String is too long for Date: " + s, ErrorCodes::TOO_LARGE_STRING_SIZE); - - return date; -} - -UInt64 stringToDateTime(const String & s) -{ - ReadBufferFromString in(s); - time_t date_time{}; - - readDateTimeText(date_time, in); - if (!in.eof()) - throw Exception("String is too long for DateTime: " + s, ErrorCodes::TOO_LARGE_STRING_SIZE); - - return UInt64(date_time); -} - -DateTime64::NativeType stringToDateTime64(const String & s, UInt32 scale) -{ - ReadBufferFromString in(s); - DateTime64 datetime64 {0}; - - readDateTime64Text(datetime64, scale, in); - if (!in.eof()) - throw Exception("String is too long for DateTime64: " + s, ErrorCodes::TOO_LARGE_STRING_SIZE); - - return datetime64.value; -} - Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const IDataType * from_type_hint) { WhichDataType which_type(type); @@ -215,35 +179,6 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID return src; } // TODO (vnemkov): extra cases for DateTime64: converting from integer, converting from Decimal - - if (src.getType() == Field::Types::String) - { - if (which_type.isDate()) - { - /// Convert 'YYYY-MM-DD' Strings to Date - return stringToDate(src.get()); - } - else if (which_type.isDateTime()) - { - /// Convert 'YYYY-MM-DD hh:mm:ss' Strings to DateTime - return stringToDateTime(src.get()); - } - else if (which_type.isDateTime64()) - { - const auto * date_time64 = typeid_cast(&type); - /// Convert 'YYYY-MM-DD hh:mm:ss.NNNNNNNNN' Strings to DateTime - return stringToDateTime64(src.get(), date_time64->getScale()); - } - else if (which_type.isUUID()) - { - return stringToUUID(src.get()); - } - else if (which_type.isEnum()) - { - /// Convert String to Enum's value - return dynamic_cast(type).castToValue(src); - } - } } else if (which_type.isStringOrFixedString()) { From ff3e5e1a2ebaba695b8ebbad8a047f9c08259f74 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 02:09:40 +0300 Subject: [PATCH 166/329] Allow implicit conversion from String in IN, VALUES and comparison #11630 --- src/Interpreters/convertFieldToType.cpp | 29 ++++++++++++++++--------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 6328ed76924..b71d2ffbaa7 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -148,7 +148,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID { return static_cast(type).getTimeZone().fromDayNum(DayNum(src.get())); } - else if (type.isValueRepresentedByNumber()) + else if (type.isValueRepresentedByNumber() && src.getType() != Field::Types::String) { if (which_type.isUInt8()) return convertNumericType(src, type); if (which_type.isUInt16()) return convertNumericType(src, type); @@ -164,9 +164,6 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID if (const auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); if (const auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); - if (!which_type.isDateOrDateTime() && !which_type.isUUID() && !which_type.isEnum()) - throw Exception{"Cannot convert field to type " + type.getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; - if (which_type.isEnum() && (src.getType() == Field::Types::UInt64 || src.getType() == Field::Types::Int64)) { /// Convert UInt64 or Int64 to Enum's value @@ -263,17 +260,29 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID return src; } + /// Conversion from string by parsing. if (src.getType() == Field::Types::String) { - const auto col = type.createColumn(); - ReadBufferFromString buffer(src.get()); - type.deserializeAsTextEscaped(*col, buffer, FormatSettings{}); + /// Promote data type to avoid overflows. Note that overflows in the largest data type are still possible. + const IDataType * type_to_parse = &type; + DataTypePtr holder; - return (*col)[0]; + if (type.canBePromoted()) + { + holder = type.promoteNumericType(); + type_to_parse = holder.get(); + } + + const auto col = type_to_parse->createColumn(); + ReadBufferFromString in_buffer(src.get()); + type_to_parse->deserializeAsWholeText(*col, in_buffer, FormatSettings{}); + if (!in_buffer.eof()) + throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "String is too long for {}: {}", type.getName(), src.get()); + + Field parsed = (*col)[0]; + return convertFieldToType(parsed, type, from_type_hint); } - - // TODO (nemkov): should we attempt to parse value using or `type.deserializeAsTextEscaped()` type.deserializeAsTextEscaped() ? throw Exception("Type mismatch in IN or VALUES section. Expected: " + type.getName() + ". Got: " + Field::Types::toString(src.getType()), ErrorCodes::TYPE_MISMATCH); } From 402484079591be39d37079b96757a964c1a0144e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 02:19:16 +0300 Subject: [PATCH 167/329] Add a test --- src/Interpreters/convertFieldToType.cpp | 10 +++++++++- ..._comparison_with_constant_string.reference | 20 +++++++++++++++++++ .../01311_comparison_with_constant_string.sql | 20 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01311_comparison_with_constant_string.reference create mode 100644 tests/queries/0_stateless/01311_comparison_with_constant_string.sql diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index b71d2ffbaa7..050be6ba956 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -275,7 +275,15 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID const auto col = type_to_parse->createColumn(); ReadBufferFromString in_buffer(src.get()); - type_to_parse->deserializeAsWholeText(*col, in_buffer, FormatSettings{}); + try + { + type_to_parse->deserializeAsWholeText(*col, in_buffer, FormatSettings{}); + } + catch (Exception & e) + { + e.addMessage(fmt::format("while converting '{}' to {}", src.get(), type.getName())); + throw; + } if (!in_buffer.eof()) throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "String is too long for {}: {}", type.getName(), src.get()); diff --git a/tests/queries/0_stateless/01311_comparison_with_constant_string.reference b/tests/queries/0_stateless/01311_comparison_with_constant_string.reference new file mode 100644 index 00000000000..48c6fc05950 --- /dev/null +++ b/tests/queries/0_stateless/01311_comparison_with_constant_string.reference @@ -0,0 +1,20 @@ +0 +1 +0 +--- +1 +0 +1 +--- +1 +0 +0 +--- +0 +--- +1 +0 +--- +--- +1 +--- diff --git a/tests/queries/0_stateless/01311_comparison_with_constant_string.sql b/tests/queries/0_stateless/01311_comparison_with_constant_string.sql new file mode 100644 index 00000000000..2cfec6ca05e --- /dev/null +++ b/tests/queries/0_stateless/01311_comparison_with_constant_string.sql @@ -0,0 +1,20 @@ +SELECT number = '1' FROM numbers(3); +SELECT '---'; +SELECT '1' != number FROM numbers(3); +SELECT '---'; +SELECT '1' > number FROM numbers(3); +SELECT '---'; +SELECT 1 = '257'; +SELECT '---'; +SELECT 1 IN (1.23, '1', 2); +SELECT 1 IN (1.23, '2', 2); +SELECT '---'; + +-- it should work but it doesn't. +SELECT 1 = '1.0'; -- { serverError 131 } +SELECT '---'; + +SELECT toDateTime('2020-06-13 01:02:03') = '2020-06-13T01:02:03'; +SELECT '---'; + +SELECT 0 = ''; -- { serverError 32 } From e2f7a41a1a640f9c4d8e402f992ae8e2fbdcf77d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 02:27:10 +0300 Subject: [PATCH 168/329] Fix error; clarify more results in test --- src/Functions/FunctionsComparison.h | 6 ++++-- .../01311_comparison_with_constant_string.reference | 12 ++++++++++++ .../01311_comparison_with_constant_string.sql | 13 +++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 9cd13df826d..91525f84c14 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -817,6 +817,7 @@ private: const DataTypePtr & left_type, const DataTypePtr & right_type, size_t input_rows_count) { /// To compare something with const string, we cast constant to appropriate type and compare as usual. + /// It is ok to throw exception if value is not convertible. /// We should deal with possible overflows, e.g. toUInt8(1) = '257' should return false. const ColumnConst * left_const = checkAndGetColumnConstStringOrFixedString(col_left_untyped); @@ -831,10 +832,11 @@ private: Field string_value = left_const ? left_const->getField() : right_const->getField(); Field converted = convertFieldToType(string_value, *type_to_compare, type_string); - /// If not possible to convert, comparison yields to false. + /// If not possible to convert, comparison with =, <, >, <=, >= yields to false and comparison with != yields to true. if (converted.isNull()) { - block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count, 0); + block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count, + std::is_same_v, NotEqualsOp>); } else { diff --git a/tests/queries/0_stateless/01311_comparison_with_constant_string.reference b/tests/queries/0_stateless/01311_comparison_with_constant_string.reference index 48c6fc05950..7865f42932d 100644 --- a/tests/queries/0_stateless/01311_comparison_with_constant_string.reference +++ b/tests/queries/0_stateless/01311_comparison_with_constant_string.reference @@ -16,5 +16,17 @@ 0 --- --- +0 +--- +1 +--- +0 +--- +0 +--- +0 +--- +0 +--- 1 --- diff --git a/tests/queries/0_stateless/01311_comparison_with_constant_string.sql b/tests/queries/0_stateless/01311_comparison_with_constant_string.sql index 2cfec6ca05e..6ca736ba146 100644 --- a/tests/queries/0_stateless/01311_comparison_with_constant_string.sql +++ b/tests/queries/0_stateless/01311_comparison_with_constant_string.sql @@ -14,6 +14,19 @@ SELECT '---'; SELECT 1 = '1.0'; -- { serverError 131 } SELECT '---'; +SELECT 1 = '257'; +SELECT '---'; +SELECT 1 != '257'; +SELECT '---'; +SELECT 1 < '257'; -- this is wrong for now +SELECT '---'; +SELECT 1 > '257'; +SELECT '---'; +SELECT 1 <= '257'; -- this is wrong for now +SELECT '---'; +SELECT 1 >= '257'; +SELECT '---'; + SELECT toDateTime('2020-06-13 01:02:03') = '2020-06-13T01:02:03'; SELECT '---'; From fed6843e64a1e7ae9f0cc1d3884a4c11976e8dda Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 03:27:00 +0300 Subject: [PATCH 169/329] Fix style --- src/Interpreters/convertFieldToType.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 050be6ba956..9c1136e5df6 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -34,7 +34,6 @@ namespace ErrorCodes extern const int ARGUMENT_OUT_OF_BOUND; extern const int TYPE_MISMATCH; extern const int TOO_LARGE_STRING_SIZE; - extern const int CANNOT_CONVERT_TYPE; } From bba0140d8fa0e42056bb9f5ba89ffe4002c4ee8f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 07:20:52 +0300 Subject: [PATCH 170/329] Fix tests --- src/Interpreters/convertFieldToType.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 9c1136e5df6..d60bb6cee6c 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -33,7 +33,6 @@ namespace ErrorCodes { extern const int ARGUMENT_OUT_OF_BOUND; extern const int TYPE_MISMATCH; - extern const int TOO_LARGE_STRING_SIZE; } @@ -284,7 +283,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID throw; } if (!in_buffer.eof()) - throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "String is too long for {}: {}", type.getName(), src.get()); + throw Exception(ErrorCodes::TYPE_MISMATCH, "Cannot convert string {} to type {}", src.get(), type.getName()); Field parsed = (*col)[0]; return convertFieldToType(parsed, type, from_type_hint); From 44221139e9f2d637a5e1a10a628692552c062291 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 07:35:50 +0300 Subject: [PATCH 171/329] Fix test --- src/DataTypes/DataTypeUUID.cpp | 7 ++++++- src/DataTypes/DataTypeUUID.h | 3 +++ src/Interpreters/convertFieldToType.cpp | 7 +++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/DataTypes/DataTypeUUID.cpp b/src/DataTypes/DataTypeUUID.cpp index 87e306cf477..94a043eb472 100644 --- a/src/DataTypes/DataTypeUUID.cpp +++ b/src/DataTypes/DataTypeUUID.cpp @@ -16,13 +16,18 @@ void DataTypeUUID::serializeText(const IColumn & column, size_t row_num, WriteBu writeText(UUID(assert_cast(column).getData()[row_num]), ostr); } -void DataTypeUUID::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void DataTypeUUID::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { UUID x; readText(x, istr); assert_cast(column).getData().push_back(x); } +void DataTypeUUID::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +{ + deserializeText(column, istr, settings); +} + void DataTypeUUID::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { serializeText(column, row_num, ostr, settings); diff --git a/src/DataTypes/DataTypeUUID.h b/src/DataTypes/DataTypeUUID.h index 75e634bc625..e9f1d22325b 100644 --- a/src/DataTypes/DataTypeUUID.h +++ b/src/DataTypes/DataTypeUUID.h @@ -17,6 +17,7 @@ public: bool equals(const IDataType & rhs) const override; void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; + void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; @@ -30,6 +31,8 @@ public: bool canBeUsedInBitOperations() const override { return true; } bool canBeInsideNullable() const override { return true; } + + bool canBePromoted() const override { return false; } }; } diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index d60bb6cee6c..21cf9422c32 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -173,6 +173,13 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID /// We don't need any conversion UInt64 is under type of Date and DateTime return src; } + + if (which_type.isUUID() && src.getType() == Field::Types::UInt128) + { + /// Already in needed type. + return src; + } + // TODO (vnemkov): extra cases for DateTime64: converting from integer, converting from Decimal } else if (which_type.isStringOrFixedString()) From 4953b5fc84c2b2b59cb8a9751553caa762961114 Mon Sep 17 00:00:00 2001 From: Bharat Nallan Date: Sat, 13 Jun 2020 21:41:55 -0700 Subject: [PATCH 172/329] remove unused imports from HTTPHandlerFactory This removes unused imports from `src/Server/HTTPHandlerFactory.cpp`: ```bash - #include - #include - #include ``` --- src/Server/HTTPHandlerFactory.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp index 2f00aa0aa72..3bcb29672e7 100644 --- a/src/Server/HTTPHandlerFactory.cpp +++ b/src/Server/HTTPHandlerFactory.cpp @@ -1,9 +1,6 @@ #include "HTTPHandlerFactory.h" -#include #include -#include -#include #include #include "HTTPHandler.h" From 3958a032acd8e3da194704505715d9be22b33787 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 08:15:29 +0300 Subject: [PATCH 173/329] Added a test --- src/AggregateFunctions/AggregateFunctionNull.cpp | 10 +++++----- ...1315_count_distinct_return_not_nullable.reference | 9 +++++++++ .../01315_count_distinct_return_not_nullable.sql | 12 ++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference create mode 100644 tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql diff --git a/src/AggregateFunctions/AggregateFunctionNull.cpp b/src/AggregateFunctions/AggregateFunctionNull.cpp index 77687f9f328..993cb93c991 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.cpp +++ b/src/AggregateFunctions/AggregateFunctionNull.cpp @@ -33,6 +33,11 @@ public: AggregateFunctionPtr transformAggregateFunction( const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params) const override { + /// Special case for 'count' function. It could be called with Nullable arguments + /// - that means - count number of calls, when all arguments are not NULL. + if (nested_function && nested_function->getName() == "count") + return std::make_shared(arguments[0], params); + bool has_nullable_types = false; bool has_null_types = false; for (const auto & arg_type : arguments) @@ -60,11 +65,6 @@ public: if (auto adapter = nested_function->getOwnNullAdapter(nested_function, arguments, params)) return adapter; - /// Special case for 'count' function. It could be called with Nullable arguments - /// - that means - count number of calls, when all arguments are not NULL. - if (nested_function->getName() == "count") - return std::make_shared(arguments[0], params); - bool return_type_is_nullable = !nested_function->returnDefaultWhenOnlyNull() && nested_function->getReturnType()->canBeInsideNullable(); bool serialize_flag = return_type_is_nullable || nested_function->returnDefaultWhenOnlyNull(); diff --git a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference new file mode 100644 index 00000000000..f8b77704aa3 --- /dev/null +++ b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference @@ -0,0 +1,9 @@ +0 +0 +0 +5 +5 +5 +0 +\N +\N diff --git a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql new file mode 100644 index 00000000000..2d9b5ef54aa --- /dev/null +++ b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql @@ -0,0 +1,12 @@ +SELECT uniq(number >= 10 ? number : NULL) FROM numbers(10); +SELECT uniqExact(number >= 10 ? number : NULL) FROM numbers(10); +SELECT count(DISTINCT number >= 10 ? number : NULL) FROM numbers(10); + +SELECT uniq(number >= 5 ? number : NULL) FROM numbers(10); +SELECT uniqExact(number >= 5 ? number : NULL) FROM numbers(10); +SELECT count(DISTINCT number >= 5 ? number : NULL) FROM numbers(10); + +SELECT count(NULL); +-- These two returns NULL for now, but we want to change them to return 0. +SELECT uniq(NULL); +SELECT count(DISTINCT NULL); From 1531f0bd0f794abba5ace7b220058d8d055dc883 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Sun, 14 Jun 2020 17:52:03 +0300 Subject: [PATCH 174/329] Update performance.html --- website/templates/index/performance.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/templates/index/performance.html b/website/templates/index/performance.html index 2a9b766c74b..61cd31a06ca 100644 --- a/website/templates/index/performance.html +++ b/website/templates/index/performance.html @@ -6,7 +6,7 @@

ClickHouse's performance exceeds comparable column-oriented database management systems currently available on the market. It processes hundreds of millions to more than a billion rows and tens of gigabytes of data per single server per second.

- Detailed comparison + Detailed comparison
From 6a439a5eb530b513cdbecfb4f4e9ce32b58bae84 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 14 Jun 2020 19:17:22 +0300 Subject: [PATCH 175/329] fixes --- src/Interpreters/InterpreterCreateQuery.cpp | 18 +++----- src/Parsers/ASTColumnDeclaration.cpp | 24 ++--------- src/Parsers/ASTColumnDeclaration.h | 3 +- src/Parsers/ParserCreateQuery.cpp | 2 +- src/Parsers/ParserCreateQuery.h | 41 +++++++------------ .../01269_creare_with_null.reference | 1 - .../01269_create_with_null.reference | 2 + .../0_stateless/01269_create_with_null.sql | 16 ++++++-- 8 files changed, 39 insertions(+), 68 deletions(-) delete mode 100644 tests/queries/0_stateless/01269_creare_with_null.reference diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 993c9e595e3..5d8c43aed0d 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -287,28 +287,22 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( const auto & col_decl = ast->as(); DataTypePtr column_type = nullptr; - if (!col_decl.is_null && col_decl.is_not) - throw Exception{"Cant use NOT without NULL", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; if (col_decl.type) { column_type = DataTypeFactory::instance().get(col_decl.type); - if (col_decl.is_not && col_decl.is_null) + if (col_decl.null_modifier) { if (column_type->isNullable()) - throw Exception{"Cant use NOT NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; - } - else if (col_decl.is_null && !col_decl.is_not) - { - if (column_type->isNullable()) - throw Exception{"Cant use NULL with Nullable", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE}; - else + throw Exception("Cant use [NOT] NULL modifier with Nullable type", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE); + if (*col_decl.null_modifier) column_type = makeNullable(column_type); } - - if (context.getSettingsRef().data_type_default_nullable && !column_type->isNullable() && !col_decl.is_not && !col_decl.is_null) + else if (context.getSettingsRef().data_type_default_nullable) + { column_type = makeNullable(column_type); + } column_names_and_types.emplace_back(col_decl.name, column_type); } diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index 08f7813ad06..5dd5fd7d526 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -18,18 +18,6 @@ ASTPtr ASTColumnDeclaration::clone() const res->children.push_back(res->type); } - if (is_null) - { - res->is_null = is_null; - res->children.push_back(res->is_null); - } - - if (is_not) - { - res->is_not = is_not; - res->children.push_back(res->is_not); - } - if (default_expression) { res->default_expression = default_expression->clone(); @@ -73,16 +61,10 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta type->formatImpl(settings, state, frame); } - if (is_not) + if (null_modifier) { - settings.ostr << ' '; - is_not->formatImpl(settings, state, frame); - } - - if (is_null) - { - settings.ostr << ' '; - is_null->formatImpl(settings, state, frame); + settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") + << (*null_modifier ? "" : "NOT ") << "NULL" << (settings.hilite ? hilite_none : ""); } if (default_expression) diff --git a/src/Parsers/ASTColumnDeclaration.h b/src/Parsers/ASTColumnDeclaration.h index 34afd771de2..ea17a8b4dfa 100644 --- a/src/Parsers/ASTColumnDeclaration.h +++ b/src/Parsers/ASTColumnDeclaration.h @@ -13,8 +13,7 @@ class ASTColumnDeclaration : public IAST public: String name; ASTPtr type; - ASTPtr is_null; - ASTPtr is_not; + std::optional null_modifier; String default_specifier; ASTPtr default_expression; ASTPtr comment; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index c54033bd27d..159b19b28c6 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -157,7 +157,7 @@ bool ParserTablePropertyDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expecte ParserIndexDeclaration index_p; ParserConstraintDeclaration constraint_p; - ParserColumnDeclaration column_p; + ParserColumnDeclaration column_p{true, true}; ASTPtr new_node = nullptr; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 9fae3d60836..a4fc60a2393 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -92,7 +92,8 @@ template class IParserColumnDeclaration : public IParserBase { public: - explicit IParserColumnDeclaration(bool require_type_ = true) : require_type(require_type_) + explicit IParserColumnDeclaration(bool require_type_ = true, bool allow_null_modifiers_ = false) + : require_type(require_type_), allow_null_modifiers(allow_null_modifiers_) { } @@ -104,6 +105,7 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; bool require_type = true; + bool allow_null_modifiers = false; }; using ParserColumnDeclaration = IParserColumnDeclaration; @@ -126,8 +128,6 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ParserStringLiteral string_literal_parser; ParserCodec codec_parser; ParserExpression expression_parser; - ParserIdentifier null_parser; - ParserCompoundIdentifier not_null_parser; /// mandatory column name ASTPtr name; @@ -139,8 +139,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E */ ASTPtr type; String default_specifier; - ASTPtr is_null; - ASTPtr is_not; + std::optional null_modifier; ASTPtr default_expression; ASTPtr comment_expression; ASTPtr codec_expression; @@ -169,19 +168,17 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (require_type && !type && !default_expression) return false; /// reject column name without type - // Pos pos_before_null = pos; - - if (s_not.check(pos, expected)) - if (s_null.check(pos, expected)) + if (type && allow_null_modifiers) + { + if (s_not.ignore(pos, expected)) { - is_not = std::make_shared("NOT"); - is_null = std::make_shared("NULL"); + if (!s_null.ignore(pos, expected)) + return false; + null_modifier.emplace(false); } - else - return false; - else - if (s_null.check(pos, expected)) - is_null = std::make_shared("NULL"); + else if (s_null.ignore(pos, expected)) + null_modifier.emplace(true); + } if (s_comment.ignore(pos, expected)) { @@ -212,17 +209,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E column_declaration->children.push_back(std::move(type)); } - if (is_null) - { - column_declaration->is_null = is_null; - column_declaration->children.push_back(std::move(is_null)); - } - - if (is_not) - { - column_declaration->is_not = is_not; - column_declaration->children.push_back(std::move(is_not)); - } + column_declaration->null_modifier = null_modifier; if (default_expression) { diff --git a/tests/queries/0_stateless/01269_creare_with_null.reference b/tests/queries/0_stateless/01269_creare_with_null.reference deleted file mode 100644 index fa7b52d9ebf..00000000000 --- a/tests/queries/0_stateless/01269_creare_with_null.reference +++ /dev/null @@ -1 +0,0 @@ -Nullable(Int32) Int32 Nullable(Int32) \ No newline at end of file diff --git a/tests/queries/0_stateless/01269_create_with_null.reference b/tests/queries/0_stateless/01269_create_with_null.reference index 7ef113393d5..739063af67f 100644 --- a/tests/queries/0_stateless/01269_create_with_null.reference +++ b/tests/queries/0_stateless/01269_create_with_null.reference @@ -1,2 +1,4 @@ Nullable(Int32) Int32 Nullable(Int32) Int32 +CREATE TABLE default.data_null\n(\n `a` Nullable(Int32), \n `b` Int32, \n `c` Nullable(Int32), \n `d` Int32\n)\nENGINE = Memory() Nullable(Int32) Int32 Nullable(Int32) Nullable(Int32) +CREATE TABLE default.set_null\n(\n `a` Nullable(Int32), \n `b` Int32, \n `c` Nullable(Int32), \n `d` Nullable(Int32)\n)\nENGINE = Memory() diff --git a/tests/queries/0_stateless/01269_create_with_null.sql b/tests/queries/0_stateless/01269_create_with_null.sql index 68fa130e0da..856b6ea75f4 100644 --- a/tests/queries/0_stateless/01269_create_with_null.sql +++ b/tests/queries/0_stateless/01269_create_with_null.sql @@ -1,6 +1,8 @@ DROP TABLE IF EXISTS data_null; DROP TABLE IF EXISTS set_null; +SET data_type_default_nullable='false'; + CREATE TABLE data_null ( a INT NULL, b INT NOT NULL, @@ -9,19 +11,20 @@ CREATE TABLE data_null ( ) engine=Memory(); -INSERT INTO data_null VALUES (1, 2, 3, 4); +INSERT INTO data_null VALUES (NULL, 2, NULL, 4); SELECT toTypeName(a), toTypeName(b), toTypeName(c), toTypeName(d) FROM data_null; +SHOW CREATE TABLE data_null; -CREATE TABLE data_null ( +CREATE TABLE data_null_error ( a Nullable(INT) NULL, b INT NOT NULL, c Nullable(INT) ) engine=Memory(); --{serverError 377} -CREATE TABLE data_null ( +CREATE TABLE data_null_error ( a INT NULL, b Nullable(INT) NOT NULL, c Nullable(INT) @@ -37,6 +40,11 @@ CREATE TABLE set_null ( ) engine=Memory(); -INSERT INTO set_null VALUES (1, 2, 3, 4); +INSERT INTO set_null VALUES (NULL, 2, NULL, NULL); SELECT toTypeName(a), toTypeName(b), toTypeName(c), toTypeName(d) FROM set_null; + +SHOW CREATE TABLE set_null; + +DROP TABLE data_null; +DROP TABLE set_null; From 394fb64a9cda376ca8dfa8bac08a4fbcfa9bf3bf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 10:44:02 +0300 Subject: [PATCH 176/329] Better way of implementation --- programs/odbc-bridge/CMakeLists.txt | 1 + .../AggregateFunctionArray.cpp | 5 +- .../AggregateFunctionCount.cpp | 8 ++- .../AggregateFunctionCount.h | 10 ++-- .../AggregateFunctionFactory.cpp | 60 +++++++++++-------- .../AggregateFunctionFactory.h | 33 +++++++--- .../AggregateFunctionForEach.cpp | 5 +- .../AggregateFunctionIf.cpp | 5 +- .../AggregateFunctionMerge.cpp | 5 +- .../AggregateFunctionNothing.h | 2 +- .../AggregateFunctionNull.cpp | 24 +++++--- .../AggregateFunctionOrFill.cpp | 1 + .../AggregateFunctionResample.cpp | 1 + .../AggregateFunctionState.cpp | 5 +- .../AggregateFunctionUniq.cpp | 6 +- .../AggregateFunctionUniq.h | 12 ---- .../AggregateFunctionUniqUpTo.cpp | 2 +- src/AggregateFunctions/IAggregateFunction.h | 20 ++++--- .../IAggregateFunctionCombinator.h | 1 + src/CMakeLists.txt | 2 +- src/Common/IFactoryWithAliases.h | 32 +++++----- src/DataStreams/tests/CMakeLists.txt | 2 +- src/DataTypes/DataTypeAggregateFunction.cpp | 3 +- .../DataTypeCustomSimpleAggregateFunction.cpp | 3 +- src/DataTypes/DataTypeFactory.cpp | 4 +- src/DataTypes/DataTypeFactory.h | 10 ++-- src/Formats/tests/CMakeLists.txt | 2 +- src/Functions/FunctionFactory.cpp | 2 +- src/Functions/FunctionFactory.h | 8 +-- src/Functions/FunctionsBitmap.h | 10 ++-- src/Functions/array/arrayReduce.cpp | 3 +- src/Functions/array/arrayReduceInRanges.cpp | 3 +- src/Interpreters/ExpressionAnalyzer.cpp | 3 +- src/Interpreters/tests/CMakeLists.txt | 4 +- src/Interpreters/tests/hash_map.cpp | 7 ++- .../Algorithms/SummingSortedAlgorithm.cpp | 3 +- .../MergeTree/registerStorageMergeTree.cpp | 5 +- src/TableFunctions/TableFunctionFactory.cpp | 2 +- src/TableFunctions/TableFunctionFactory.h | 9 ++- ...unt_distinct_return_not_nullable.reference | 4 +- ...315_count_distinct_return_not_nullable.sql | 1 - .../CMakeLists.txt | 2 +- .../CMakeLists.txt | 2 +- 43 files changed, 196 insertions(+), 136 deletions(-) diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index ab8d94f2a0c..628f9ee018a 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -14,6 +14,7 @@ set (CLICKHOUSE_ODBC_BRIDGE_SOURCES set (CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE clickhouse_parsers + clickhouse_aggregate_functions daemon dbms Poco::Data diff --git a/src/AggregateFunctions/AggregateFunctionArray.cpp b/src/AggregateFunctions/AggregateFunctionArray.cpp index ced95185263..7fe4f1f448b 100644 --- a/src/AggregateFunctions/AggregateFunctionArray.cpp +++ b/src/AggregateFunctions/AggregateFunctionArray.cpp @@ -36,7 +36,10 @@ public: } AggregateFunctionPtr transformAggregateFunction( - const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array &) const override + const AggregateFunctionPtr & nested_function, + const AggregateFunctionProperties &, + const DataTypes & arguments, + const Array &) const override { return std::make_shared(nested_function, arguments); } diff --git a/src/AggregateFunctions/AggregateFunctionCount.cpp b/src/AggregateFunctions/AggregateFunctionCount.cpp index 6c22fec87a2..b00adaa0f1a 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.cpp +++ b/src/AggregateFunctions/AggregateFunctionCount.cpp @@ -7,6 +7,12 @@ namespace DB { +AggregateFunctionPtr AggregateFunctionCount::getOwnNullAdapter( + const AggregateFunctionPtr &, const DataTypes & types, const Array & params) const +{ + return std::make_shared(types[0], params); +} + namespace { @@ -22,7 +28,7 @@ AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, cons void registerAggregateFunctionCount(AggregateFunctionFactory & factory) { - factory.registerFunction("count", createAggregateFunctionCount, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("count", {createAggregateFunctionCount, {true}}, AggregateFunctionFactory::CaseInsensitive); } } diff --git a/src/AggregateFunctions/AggregateFunctionCount.h b/src/AggregateFunctions/AggregateFunctionCount.h index e54f014f7a4..feb5725d9f1 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.h +++ b/src/AggregateFunctions/AggregateFunctionCount.h @@ -68,16 +68,14 @@ public: data(place).count = new_count; } - /// The function returns non-Nullable type even when wrapped with Null combinator. - bool returnDefaultWhenOnlyNull() const override - { - return true; - } + AggregateFunctionPtr getOwnNullAdapter( + const AggregateFunctionPtr &, const DataTypes & types, const Array & params) const override; }; /// Simply count number of not-NULL values. -class AggregateFunctionCountNotNullUnary final : public IAggregateFunctionDataHelper +class AggregateFunctionCountNotNullUnary final + : public IAggregateFunctionDataHelper { public: AggregateFunctionCountNotNullUnary(const DataTypePtr & argument, const Array & params) diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index 3982c48700b..7ff52fe0f70 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -29,18 +29,18 @@ namespace ErrorCodes } -void AggregateFunctionFactory::registerFunction(const String & name, Creator creator, CaseSensitiveness case_sensitiveness) +void AggregateFunctionFactory::registerFunction(const String & name, Value creator_with_properties, CaseSensitiveness case_sensitiveness) { - if (creator == nullptr) + if (creator_with_properties.creator == nullptr) throw Exception("AggregateFunctionFactory: the aggregate function " + name + " has been provided " " a null constructor", ErrorCodes::LOGICAL_ERROR); - if (!aggregate_functions.emplace(name, creator).second) + if (!aggregate_functions.emplace(name, creator_with_properties).second) throw Exception("AggregateFunctionFactory: the aggregate function name '" + name + "' is not unique", ErrorCodes::LOGICAL_ERROR); if (case_sensitiveness == CaseInsensitive - && !case_insensitive_aggregate_functions.emplace(Poco::toLower(name), creator).second) + && !case_insensitive_aggregate_functions.emplace(Poco::toLower(name), creator_with_properties).second) throw Exception("AggregateFunctionFactory: the case insensitive aggregate function name '" + name + "' is not unique", ErrorCodes::LOGICAL_ERROR); } @@ -59,6 +59,7 @@ AggregateFunctionPtr AggregateFunctionFactory::get( const String & name, const DataTypes & argument_types, const Array & parameters, + AggregateFunctionProperties & out_properties, int recursion_level) const { auto type_without_low_cardinality = convertLowCardinalityTypesToNested(argument_types); @@ -76,18 +77,11 @@ AggregateFunctionPtr AggregateFunctionFactory::get( DataTypes nested_types = combinator->transformArguments(type_without_low_cardinality); Array nested_parameters = combinator->transformParameters(parameters); - AggregateFunctionPtr nested_function; - - /// A little hack - if we have NULL arguments, don't even create nested function. - /// Combinator will check if nested_function was created. - if (name == "count" || std::none_of(type_without_low_cardinality.begin(), type_without_low_cardinality.end(), - [](const auto & type) { return type->onlyNull(); })) - nested_function = getImpl(name, nested_types, nested_parameters, recursion_level); - - return combinator->transformAggregateFunction(nested_function, type_without_low_cardinality, parameters); + AggregateFunctionPtr nested_function = getImpl(name, nested_types, nested_parameters, out_properties, recursion_level); + return combinator->transformAggregateFunction(nested_function, out_properties, type_without_low_cardinality, parameters); } - auto res = getImpl(name, type_without_low_cardinality, parameters, recursion_level); + auto res = getImpl(name, type_without_low_cardinality, parameters, out_properties, recursion_level); if (!res) throw Exception("Logical error: AggregateFunctionFactory returned nullptr", ErrorCodes::LOGICAL_ERROR); return res; @@ -98,19 +92,37 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( const String & name_param, const DataTypes & argument_types, const Array & parameters, + AggregateFunctionProperties & out_properties, int recursion_level) const { String name = getAliasToOrName(name_param); + Value found; + /// Find by exact match. if (auto it = aggregate_functions.find(name); it != aggregate_functions.end()) - return it->second(name, argument_types, parameters); - + { + found = it->second; + } /// Find by case-insensitive name. /// Combinators cannot apply for case insensitive (SQL-style) aggregate function names. Only for native names. - if (recursion_level == 0) + else if (recursion_level == 0) { - if (auto it = case_insensitive_aggregate_functions.find(Poco::toLower(name)); it != case_insensitive_aggregate_functions.end()) - return it->second(name, argument_types, parameters); + if (auto jt = case_insensitive_aggregate_functions.find(Poco::toLower(name)); jt != case_insensitive_aggregate_functions.end()) + found = jt->second; + } + + if (found.creator) + { + out_properties = found.properties; + + /// The case when aggregate function should return NULL on NULL arguments. This case is handled in "get" method. + if (!out_properties.returns_default_when_only_null + && std::any_of(argument_types.begin(), argument_types.end(), [](const auto & type) { return type->onlyNull(); })) + { + return nullptr; + } + + return found.creator(name, argument_types, parameters); } /// Combinators of aggregate functions. @@ -126,9 +138,8 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( DataTypes nested_types = combinator->transformArguments(argument_types); Array nested_parameters = combinator->transformParameters(parameters); - AggregateFunctionPtr nested_function = get(nested_name, nested_types, nested_parameters, recursion_level + 1); - - return combinator->transformAggregateFunction(nested_function, argument_types, parameters); + AggregateFunctionPtr nested_function = get(nested_name, nested_types, nested_parameters, out_properties, recursion_level + 1); + return combinator->transformAggregateFunction(nested_function, out_properties, argument_types, parameters); } auto hints = this->getHints(name); @@ -140,10 +151,11 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( } -AggregateFunctionPtr AggregateFunctionFactory::tryGet(const String & name, const DataTypes & argument_types, const Array & parameters) const +AggregateFunctionPtr AggregateFunctionFactory::tryGet( + const String & name, const DataTypes & argument_types, const Array & parameters, AggregateFunctionProperties & out_properties) const { return isAggregateFunctionName(name) - ? get(name, argument_types, parameters) + ? get(name, argument_types, parameters, out_properties) : nullptr; } diff --git a/src/AggregateFunctions/AggregateFunctionFactory.h b/src/AggregateFunctions/AggregateFunctionFactory.h index 6e755cc9e8c..ab45fcf683f 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.h +++ b/src/AggregateFunctions/AggregateFunctionFactory.h @@ -26,34 +26,50 @@ using DataTypes = std::vector; */ using AggregateFunctionCreator = std::function; +struct AggregateFunctionWithProperties +{ + AggregateFunctionCreator creator; + AggregateFunctionProperties properties; + + AggregateFunctionWithProperties() = default; + AggregateFunctionWithProperties(const AggregateFunctionWithProperties &) = default; + + template > * = nullptr> + AggregateFunctionWithProperties(Creator creator_, AggregateFunctionProperties properties_ = {}) + : creator(std::forward(creator_)), properties(std::move(properties_)) + { + } +}; + /** Creates an aggregate function by name. */ -class AggregateFunctionFactory final : private boost::noncopyable, public IFactoryWithAliases +class AggregateFunctionFactory final : private boost::noncopyable, public IFactoryWithAliases { public: - static AggregateFunctionFactory & instance(); /// Register a function by its name. /// No locking, you must register all functions before usage of get. void registerFunction( const String & name, - Creator creator, + Value creator, CaseSensitiveness case_sensitiveness = CaseSensitive); /// Throws an exception if not found. AggregateFunctionPtr get( const String & name, const DataTypes & argument_types, - const Array & parameters = {}, + const Array & parameters, + AggregateFunctionProperties & out_properties, int recursion_level = 0) const; /// Returns nullptr if not found. AggregateFunctionPtr tryGet( const String & name, const DataTypes & argument_types, - const Array & parameters = {}) const; + const Array & parameters, + AggregateFunctionProperties & out_properties) const; bool isAggregateFunctionName(const String & name, int recursion_level = 0) const; @@ -62,19 +78,20 @@ private: const String & name, const DataTypes & argument_types, const Array & parameters, + AggregateFunctionProperties & out_properties, int recursion_level) const; private: - using AggregateFunctions = std::unordered_map; + using AggregateFunctions = std::unordered_map; AggregateFunctions aggregate_functions; /// Case insensitive aggregate functions will be additionally added here with lowercased name. AggregateFunctions case_insensitive_aggregate_functions; - const AggregateFunctions & getCreatorMap() const override { return aggregate_functions; } + const AggregateFunctions & getMap() const override { return aggregate_functions; } - const AggregateFunctions & getCaseInsensitiveCreatorMap() const override { return case_insensitive_aggregate_functions; } + const AggregateFunctions & getCaseInsensitiveMap() const override { return case_insensitive_aggregate_functions; } String getFactoryName() const override { return "AggregateFunctionFactory"; } diff --git a/src/AggregateFunctions/AggregateFunctionForEach.cpp b/src/AggregateFunctions/AggregateFunctionForEach.cpp index 775dab2dcd9..693bc6839fa 100644 --- a/src/AggregateFunctions/AggregateFunctionForEach.cpp +++ b/src/AggregateFunctions/AggregateFunctionForEach.cpp @@ -33,7 +33,10 @@ public: } AggregateFunctionPtr transformAggregateFunction( - const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array &) const override + const AggregateFunctionPtr & nested_function, + const AggregateFunctionProperties &, + const DataTypes & arguments, + const Array &) const override { return std::make_shared(nested_function, arguments); } diff --git a/src/AggregateFunctions/AggregateFunctionIf.cpp b/src/AggregateFunctions/AggregateFunctionIf.cpp index cb5f9f15b1c..19a175de911 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.cpp +++ b/src/AggregateFunctions/AggregateFunctionIf.cpp @@ -31,7 +31,10 @@ public: } AggregateFunctionPtr transformAggregateFunction( - const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array &) const override + const AggregateFunctionPtr & nested_function, + const AggregateFunctionProperties &, + const DataTypes & arguments, + const Array &) const override { return std::make_shared(nested_function, arguments); } diff --git a/src/AggregateFunctions/AggregateFunctionMerge.cpp b/src/AggregateFunctions/AggregateFunctionMerge.cpp index 05d941844d9..2ce3f0e11f6 100644 --- a/src/AggregateFunctions/AggregateFunctionMerge.cpp +++ b/src/AggregateFunctions/AggregateFunctionMerge.cpp @@ -34,7 +34,10 @@ public: } AggregateFunctionPtr transformAggregateFunction( - const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array &) const override + const AggregateFunctionPtr & nested_function, + const AggregateFunctionProperties &, + const DataTypes & arguments, + const Array &) const override { const DataTypePtr & argument = arguments[0]; diff --git a/src/AggregateFunctions/AggregateFunctionNothing.h b/src/AggregateFunctions/AggregateFunctionNothing.h index 511dbbecd38..b3206f6db6e 100644 --- a/src/AggregateFunctions/AggregateFunctionNothing.h +++ b/src/AggregateFunctions/AggregateFunctionNothing.h @@ -25,7 +25,7 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared(std::make_shared()); + return argument_types.front(); } void create(AggregateDataPtr) const override diff --git a/src/AggregateFunctions/AggregateFunctionNull.cpp b/src/AggregateFunctions/AggregateFunctionNull.cpp index 993cb93c991..85d960eae62 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.cpp +++ b/src/AggregateFunctions/AggregateFunctionNull.cpp @@ -31,13 +31,11 @@ public: } AggregateFunctionPtr transformAggregateFunction( - const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params) const override + const AggregateFunctionPtr & nested_function, + const AggregateFunctionProperties & properties, + const DataTypes & arguments, + const Array & params) const override { - /// Special case for 'count' function. It could be called with Nullable arguments - /// - that means - count number of calls, when all arguments are not NULL. - if (nested_function && nested_function->getName() == "count") - return std::make_shared(arguments[0], params); - bool has_nullable_types = false; bool has_null_types = false; for (const auto & arg_type : arguments) @@ -58,15 +56,23 @@ public: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (has_null_types) - return std::make_shared(arguments, params); + { + std::cerr << properties.returns_default_when_only_null << "\n"; + + /// Currently the only functions that returns not-NULL on all NULL arguments are count and uniq, and they returns UInt64. + if (properties.returns_default_when_only_null) + return std::make_shared(DataTypes{std::make_shared()}, params); + else + return std::make_shared(arguments, params); + } assert(nested_function); if (auto adapter = nested_function->getOwnNullAdapter(nested_function, arguments, params)) return adapter; - bool return_type_is_nullable = !nested_function->returnDefaultWhenOnlyNull() && nested_function->getReturnType()->canBeInsideNullable(); - bool serialize_flag = return_type_is_nullable || nested_function->returnDefaultWhenOnlyNull(); + bool return_type_is_nullable = !properties.returns_default_when_only_null && nested_function->getReturnType()->canBeInsideNullable(); + bool serialize_flag = return_type_is_nullable || properties.returns_default_when_only_null; if (arguments.size() == 1) { diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.cpp b/src/AggregateFunctions/AggregateFunctionOrFill.cpp index b9cc2f9b8b7..ce8fc8d9ca5 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.cpp +++ b/src/AggregateFunctions/AggregateFunctionOrFill.cpp @@ -21,6 +21,7 @@ public: AggregateFunctionPtr transformAggregateFunction( const AggregateFunctionPtr & nested_function, + const AggregateFunctionProperties &, const DataTypes & arguments, const Array & params) const override { diff --git a/src/AggregateFunctions/AggregateFunctionResample.cpp b/src/AggregateFunctions/AggregateFunctionResample.cpp index d8d13e22120..389c9048918 100644 --- a/src/AggregateFunctions/AggregateFunctionResample.cpp +++ b/src/AggregateFunctions/AggregateFunctionResample.cpp @@ -43,6 +43,7 @@ public: AggregateFunctionPtr transformAggregateFunction( const AggregateFunctionPtr & nested_function, + const AggregateFunctionProperties &, const DataTypes & arguments, const Array & params) const override { diff --git a/src/AggregateFunctions/AggregateFunctionState.cpp b/src/AggregateFunctions/AggregateFunctionState.cpp index fd92953d114..9d1c677c0ff 100644 --- a/src/AggregateFunctions/AggregateFunctionState.cpp +++ b/src/AggregateFunctions/AggregateFunctionState.cpp @@ -24,7 +24,10 @@ public: } AggregateFunctionPtr transformAggregateFunction( - const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params) const override + const AggregateFunctionPtr & nested_function, + const AggregateFunctionProperties &, + const DataTypes & arguments, + const Array & params) const override { return std::make_shared(nested_function, arguments, params); } diff --git a/src/AggregateFunctions/AggregateFunctionUniq.cpp b/src/AggregateFunctions/AggregateFunctionUniq.cpp index 1d079550124..40742ae336e 100644 --- a/src/AggregateFunctions/AggregateFunctionUniq.cpp +++ b/src/AggregateFunctions/AggregateFunctionUniq.cpp @@ -123,13 +123,13 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory) { factory.registerFunction("uniq", - createAggregateFunctionUniq); + {createAggregateFunctionUniq, {true}}); factory.registerFunction("uniqHLL12", - createAggregateFunctionUniq); + {createAggregateFunctionUniq, {true}}); factory.registerFunction("uniqExact", - createAggregateFunctionUniq>); + {createAggregateFunctionUniq>, {true}}); } } diff --git a/src/AggregateFunctions/AggregateFunctionUniq.h b/src/AggregateFunctions/AggregateFunctionUniq.h index 1588611b8a2..334e809ebe7 100644 --- a/src/AggregateFunctions/AggregateFunctionUniq.h +++ b/src/AggregateFunctions/AggregateFunctionUniq.h @@ -244,12 +244,6 @@ public: { assert_cast(to).getData().push_back(this->data(place).set.size()); } - - /// The function returns non-Nullable type even when wrapped with Null combinator. - bool returnDefaultWhenOnlyNull() const override - { - return true; - } }; @@ -304,12 +298,6 @@ public: { assert_cast(to).getData().push_back(this->data(place).set.size()); } - - /// The function returns non-Nullable type even when wrapped with Null combinator. - bool returnDefaultWhenOnlyNull() const override - { - return true; - } }; } diff --git a/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp b/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp index a9a8ae0eaf3..9befc515de6 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp +++ b/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp @@ -85,7 +85,7 @@ AggregateFunctionPtr createAggregateFunctionUniqUpTo(const std::string & name, c void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory & factory) { - factory.registerFunction("uniqUpTo", createAggregateFunctionUniqUpTo); + factory.registerFunction("uniqUpTo", {createAggregateFunctionUniqUpTo, {true}}); } } diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 439a5e07c2e..5f4291dd21d 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -166,17 +166,12 @@ public: * nested_function is a smart pointer to this aggregate function itself. * arguments and params are for nested_function. */ - virtual AggregateFunctionPtr getOwnNullAdapter(const AggregateFunctionPtr & /*nested_function*/, const DataTypes & /*arguments*/, const Array & /*params*/) const + virtual AggregateFunctionPtr getOwnNullAdapter( + const AggregateFunctionPtr & /*nested_function*/, const DataTypes & /*arguments*/, const Array & /*params*/) const { return nullptr; } - /** When the function is wrapped with Null combinator, - * should we return Nullable type with NULL when no values were aggregated - * or we should return non-Nullable type with default value (example: count, countDistinct). - */ - virtual bool returnDefaultWhenOnlyNull() const { return false; } - const DataTypes & getArgumentTypes() const { return argument_types; } const Array & getParameters() const { return parameters; } @@ -286,4 +281,15 @@ public: }; +/// Properties of aggregate function that are independent of argument types and parameters. +struct AggregateFunctionProperties +{ + /** When the function is wrapped with Null combinator, + * should we return Nullable type with NULL when no values were aggregated + * or we should return non-Nullable type with default value (example: count, countDistinct). + */ + bool returns_default_when_only_null = false; +}; + + } diff --git a/src/AggregateFunctions/IAggregateFunctionCombinator.h b/src/AggregateFunctions/IAggregateFunctionCombinator.h index 03e2766dc2c..89c313567a3 100644 --- a/src/AggregateFunctions/IAggregateFunctionCombinator.h +++ b/src/AggregateFunctions/IAggregateFunctionCombinator.h @@ -59,6 +59,7 @@ public: */ virtual AggregateFunctionPtr transformAggregateFunction( const AggregateFunctionPtr & nested_function, + const AggregateFunctionProperties & properties, const DataTypes & arguments, const Array & params) const = 0; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fe223373cf3..321bba1139a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -381,6 +381,6 @@ if (ENABLE_TESTS AND USE_GTEST) -Wno-gnu-zero-variadic-macro-arguments ) - target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils) + target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_aggregate_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils) add_check(unit_tests_dbms) endif () diff --git a/src/Common/IFactoryWithAliases.h b/src/Common/IFactoryWithAliases.h index 64703e51082..994b2c1a02c 100644 --- a/src/Common/IFactoryWithAliases.h +++ b/src/Common/IFactoryWithAliases.h @@ -16,14 +16,14 @@ namespace ErrorCodes } /** If stored objects may have several names (aliases) - * this interface may be helpful - * template parameter is available as Creator - */ -template -class IFactoryWithAliases : public IHints<2, IFactoryWithAliases> + * this interface may be helpful + * template parameter is available as Value + */ +template +class IFactoryWithAliases : public IHints<2, IFactoryWithAliases> { protected: - using Creator = CreatorFunc; + using Value = ValueType; String getAliasToOrName(const String & name) const { @@ -43,13 +43,13 @@ public: CaseInsensitive }; - /** Register additional name for creator - * real_name have to be already registered. - */ + /** Register additional name for value + * real_name have to be already registered. + */ void registerAlias(const String & alias_name, const String & real_name, CaseSensitiveness case_sensitiveness = CaseSensitive) { - const auto & creator_map = getCreatorMap(); - const auto & case_insensitive_creator_map = getCaseInsensitiveCreatorMap(); + const auto & creator_map = getMap(); + const auto & case_insensitive_creator_map = getCaseInsensitiveMap(); const String factory_name = getFactoryName(); String real_dict_name; @@ -80,7 +80,7 @@ public: { std::vector result; auto getter = [](const auto & pair) { return pair.first; }; - std::transform(getCreatorMap().begin(), getCreatorMap().end(), std::back_inserter(result), getter); + std::transform(getMap().begin(), getMap().end(), std::back_inserter(result), getter); std::transform(aliases.begin(), aliases.end(), std::back_inserter(result), getter); return result; } @@ -88,7 +88,7 @@ public: bool isCaseInsensitive(const String & name) const { String name_lowercase = Poco::toLower(name); - return getCaseInsensitiveCreatorMap().count(name_lowercase) || case_insensitive_aliases.count(name_lowercase); + return getCaseInsensitiveMap().count(name_lowercase) || case_insensitive_aliases.count(name_lowercase); } const String & aliasTo(const String & name) const @@ -109,11 +109,11 @@ public: virtual ~IFactoryWithAliases() override {} private: - using InnerMap = std::unordered_map; // name -> creator + using InnerMap = std::unordered_map; // name -> creator using AliasMap = std::unordered_map; // alias -> original type - virtual const InnerMap & getCreatorMap() const = 0; - virtual const InnerMap & getCaseInsensitiveCreatorMap() const = 0; + virtual const InnerMap & getMap() const = 0; + virtual const InnerMap & getCaseInsensitiveMap() const = 0; virtual String getFactoryName() const = 0; /// Alias map to data_types from previous two maps diff --git a/src/DataStreams/tests/CMakeLists.txt b/src/DataStreams/tests/CMakeLists.txt index 14db417b71c..d01c79aee5f 100644 --- a/src/DataStreams/tests/CMakeLists.txt +++ b/src/DataStreams/tests/CMakeLists.txt @@ -1,4 +1,4 @@ set(SRCS) add_executable (finish_sorting_stream finish_sorting_stream.cpp ${SRCS}) -target_link_libraries (finish_sorting_stream PRIVATE dbms) +target_link_libraries (finish_sorting_stream PRIVATE clickhouse_aggregate_functions dbms) diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 59811b1cd55..fdb17606f78 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -392,7 +392,8 @@ static DataTypePtr create(const ASTPtr & arguments) if (function_name.empty()) throw Exception("Logical error: empty name of aggregate function passed", ErrorCodes::LOGICAL_ERROR); - function = AggregateFunctionFactory::instance().get(function_name, argument_types, params_row); + AggregateFunctionProperties properties; + function = AggregateFunctionFactory::instance().get(function_name, argument_types, params_row, properties); return std::make_shared(function, argument_types, params_row); } diff --git a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp index 2ddce184cce..157192642ba 100644 --- a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp +++ b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp @@ -110,7 +110,8 @@ static std::pair create(const ASTPtr & argum if (function_name.empty()) throw Exception("Logical error: empty name of aggregate function passed", ErrorCodes::LOGICAL_ERROR); - function = AggregateFunctionFactory::instance().get(function_name, argument_types, params_row); + AggregateFunctionProperties properties; + function = AggregateFunctionFactory::instance().get(function_name, argument_types, params_row, properties); // check function if (std::find(std::begin(supported_functions), std::end(supported_functions), function->getName()) == std::end(supported_functions)) diff --git a/src/DataTypes/DataTypeFactory.cpp b/src/DataTypes/DataTypeFactory.cpp index 880f25d009d..69dbed10ccc 100644 --- a/src/DataTypes/DataTypeFactory.cpp +++ b/src/DataTypes/DataTypeFactory.cpp @@ -80,7 +80,7 @@ DataTypePtr DataTypeFactory::get(const String & family_name_param, const ASTPtr } -void DataTypeFactory::registerDataType(const String & family_name, Creator creator, CaseSensitiveness case_sensitiveness) +void DataTypeFactory::registerDataType(const String & family_name, Value creator, CaseSensitiveness case_sensitiveness) { if (creator == nullptr) throw Exception("DataTypeFactory: the data type family " + family_name + " has been provided " @@ -136,7 +136,7 @@ void DataTypeFactory::registerSimpleDataTypeCustom(const String &name, SimpleCre }, case_sensitiveness); } -const DataTypeFactory::Creator& DataTypeFactory::findCreatorByName(const String & family_name) const +const DataTypeFactory::Value & DataTypeFactory::findCreatorByName(const String & family_name) const { { DataTypesDictionary::const_iterator it = data_types.find(family_name); diff --git a/src/DataTypes/DataTypeFactory.h b/src/DataTypes/DataTypeFactory.h index 6bf09d31727..67b72945acc 100644 --- a/src/DataTypes/DataTypeFactory.h +++ b/src/DataTypes/DataTypeFactory.h @@ -23,7 +23,7 @@ class DataTypeFactory final : private boost::noncopyable, public IFactoryWithAli { private: using SimpleCreator = std::function; - using DataTypesDictionary = std::unordered_map; + using DataTypesDictionary = std::unordered_map; using CreatorWithCustom = std::function(const ASTPtr & parameters)>; using SimpleCreatorWithCustom = std::function()>; @@ -35,7 +35,7 @@ public: DataTypePtr get(const ASTPtr & ast) const; /// Register a type family by its name. - void registerDataType(const String & family_name, Creator creator, CaseSensitiveness case_sensitiveness = CaseSensitive); + void registerDataType(const String & family_name, Value creator, CaseSensitiveness case_sensitiveness = CaseSensitive); /// Register a simple data type, that have no parameters. void registerSimpleDataType(const String & name, SimpleCreator creator, CaseSensitiveness case_sensitiveness = CaseSensitive); @@ -47,7 +47,7 @@ public: void registerSimpleDataTypeCustom(const String & name, SimpleCreatorWithCustom creator, CaseSensitiveness case_sensitiveness = CaseSensitive); private: - const Creator& findCreatorByName(const String & family_name) const; + const Value & findCreatorByName(const String & family_name) const; private: DataTypesDictionary data_types; @@ -57,9 +57,9 @@ private: DataTypeFactory(); - const DataTypesDictionary & getCreatorMap() const override { return data_types; } + const DataTypesDictionary & getMap() const override { return data_types; } - const DataTypesDictionary & getCaseInsensitiveCreatorMap() const override { return case_insensitive_data_types; } + const DataTypesDictionary & getCaseInsensitiveMap() const override { return case_insensitive_data_types; } String getFactoryName() const override { return "DataTypeFactory"; } }; diff --git a/src/Formats/tests/CMakeLists.txt b/src/Formats/tests/CMakeLists.txt index 187700dff72..e1cb7604fab 100644 --- a/src/Formats/tests/CMakeLists.txt +++ b/src/Formats/tests/CMakeLists.txt @@ -1,4 +1,4 @@ set(SRCS ) add_executable (tab_separated_streams tab_separated_streams.cpp ${SRCS}) -target_link_libraries (tab_separated_streams PRIVATE dbms) +target_link_libraries (tab_separated_streams PRIVATE clickhouse_aggregate_functions dbms) diff --git a/src/Functions/FunctionFactory.cpp b/src/Functions/FunctionFactory.cpp index 63f12188771..fbc8e11a9c9 100644 --- a/src/Functions/FunctionFactory.cpp +++ b/src/Functions/FunctionFactory.cpp @@ -20,7 +20,7 @@ namespace ErrorCodes void FunctionFactory::registerFunction(const std::string & name, - Creator creator, + Value creator, CaseSensitiveness case_sensitiveness) { if (!functions.emplace(name, creator).second) diff --git a/src/Functions/FunctionFactory.h b/src/Functions/FunctionFactory.h index ccaf2044693..7990e78daf8 100644 --- a/src/Functions/FunctionFactory.h +++ b/src/Functions/FunctionFactory.h @@ -53,7 +53,7 @@ public: FunctionOverloadResolverImplPtr tryGetImpl(const std::string & name, const Context & context) const; private: - using Functions = std::unordered_map; + using Functions = std::unordered_map; Functions functions; Functions case_insensitive_functions; @@ -64,9 +64,9 @@ private: return std::make_unique(Function::create(context)); } - const Functions & getCreatorMap() const override { return functions; } + const Functions & getMap() const override { return functions; } - const Functions & getCaseInsensitiveCreatorMap() const override { return case_insensitive_functions; } + const Functions & getCaseInsensitiveMap() const override { return case_insensitive_functions; } String getFactoryName() const override { return "FunctionFactory"; } @@ -74,7 +74,7 @@ private: /// No locking, you must register all functions before usage of get. void registerFunction( const std::string & name, - Creator creator, + Value creator, CaseSensitiveness case_sensitiveness = CaseSensitive); }; diff --git a/src/Functions/FunctionsBitmap.h b/src/Functions/FunctionsBitmap.h index bf84bfbe47e..868bf8095a4 100644 --- a/src/Functions/FunctionsBitmap.h +++ b/src/Functions/FunctionsBitmap.h @@ -113,8 +113,9 @@ public: auto nested_type = array_type->getNestedType(); DataTypes argument_types = {nested_type}; Array params_row; - AggregateFunctionPtr bitmap_function - = AggregateFunctionFactory::instance().get(AggregateFunctionGroupBitmapData::name(), argument_types, params_row); + AggregateFunctionProperties properties; + AggregateFunctionPtr bitmap_function = AggregateFunctionFactory::instance().get( + AggregateFunctionGroupBitmapData::name(), argument_types, params_row, properties); return std::make_shared(bitmap_function, argument_types, params_row); } @@ -156,8 +157,9 @@ private: // output data Array params_row; - AggregateFunctionPtr bitmap_function - = AggregateFunctionFactory::instance().get(AggregateFunctionGroupBitmapData::name(), argument_types, params_row); + AggregateFunctionProperties properties; + AggregateFunctionPtr bitmap_function = AggregateFunctionFactory::instance().get( + AggregateFunctionGroupBitmapData::name(), argument_types, params_row, properties); auto col_to = ColumnAggregateFunction::create(bitmap_function); col_to->reserve(offsets.size()); diff --git a/src/Functions/array/arrayReduce.cpp b/src/Functions/array/arrayReduce.cpp index 8d44acc82f5..2b37965260f 100644 --- a/src/Functions/array/arrayReduce.cpp +++ b/src/Functions/array/arrayReduce.cpp @@ -97,7 +97,8 @@ DataTypePtr FunctionArrayReduce::getReturnTypeImpl(const ColumnsWithTypeAndName getAggregateFunctionNameAndParametersArray(aggregate_function_name_with_params, aggregate_function_name, params_row, "function " + getName()); - aggregate_function = AggregateFunctionFactory::instance().get(aggregate_function_name, argument_types, params_row); + AggregateFunctionProperties properties; + aggregate_function = AggregateFunctionFactory::instance().get(aggregate_function_name, argument_types, params_row, properties); } return aggregate_function->getReturnType(); diff --git a/src/Functions/array/arrayReduceInRanges.cpp b/src/Functions/array/arrayReduceInRanges.cpp index 2dd0cd56343..c3c65c4d9e5 100644 --- a/src/Functions/array/arrayReduceInRanges.cpp +++ b/src/Functions/array/arrayReduceInRanges.cpp @@ -115,7 +115,8 @@ DataTypePtr FunctionArrayReduceInRanges::getReturnTypeImpl(const ColumnsWithType getAggregateFunctionNameAndParametersArray(aggregate_function_name_with_params, aggregate_function_name, params_row, "function " + getName()); - aggregate_function = AggregateFunctionFactory::instance().get(aggregate_function_name, argument_types, params_row); + AggregateFunctionProperties properties; + aggregate_function = AggregateFunctionFactory::instance().get(aggregate_function_name, argument_types, params_row, properties); } return std::make_shared(aggregate_function->getReturnType()); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index ecfa011f1c8..4c2a8b3dcea 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -420,8 +420,9 @@ bool ExpressionAnalyzer::makeAggregateDescriptions(ExpressionActionsPtr & action aggregate.argument_names[i] = name; } + AggregateFunctionProperties properties; aggregate.parameters = (node->parameters) ? getAggregateFunctionParametersArray(node->parameters) : Array(); - aggregate.function = AggregateFunctionFactory::instance().get(node->name, types, aggregate.parameters); + aggregate.function = AggregateFunctionFactory::instance().get(node->name, types, aggregate.parameters, properties); aggregate_descriptions.push_back(aggregate); } diff --git a/src/Interpreters/tests/CMakeLists.txt b/src/Interpreters/tests/CMakeLists.txt index 324a38b1a17..4ab7da014e4 100644 --- a/src/Interpreters/tests/CMakeLists.txt +++ b/src/Interpreters/tests/CMakeLists.txt @@ -34,11 +34,11 @@ target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARSEHAS target_link_libraries (two_level_hash_map PRIVATE dbms) add_executable (in_join_subqueries_preprocessor in_join_subqueries_preprocessor.cpp) -target_link_libraries (in_join_subqueries_preprocessor PRIVATE dbms clickhouse_parsers) +target_link_libraries (in_join_subqueries_preprocessor PRIVATE clickhouse_aggregate_functions dbms clickhouse_parsers) add_check(in_join_subqueries_preprocessor) add_executable (users users.cpp) -target_link_libraries (users PRIVATE dbms clickhouse_common_config) +target_link_libraries (users PRIVATE clickhouse_aggregate_functions dbms clickhouse_common_config) if (OS_LINUX) add_executable (internal_iotop internal_iotop.cpp) diff --git a/src/Interpreters/tests/hash_map.cpp b/src/Interpreters/tests/hash_map.cpp index 8ddbd3b5886..dc87fd9ddde 100644 --- a/src/Interpreters/tests/hash_map.cpp +++ b/src/Interpreters/tests/hash_map.cpp @@ -103,9 +103,10 @@ int main(int argc, char ** argv) std::vector data(n); Value value; - AggregateFunctionPtr func_count = factory.get("count", data_types_empty); - AggregateFunctionPtr func_avg = factory.get("avg", data_types_uint64); - AggregateFunctionPtr func_uniq = factory.get("uniq", data_types_uint64); + AggregateFunctionProperties properties; + AggregateFunctionPtr func_count = factory.get("count", data_types_empty, {}, properties); + AggregateFunctionPtr func_avg = factory.get("avg", data_types_uint64, {}, properties); + AggregateFunctionPtr func_uniq = factory.get("uniq", data_types_uint64, {}, properties); #define INIT \ { \ diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 89154044ae5..8be4aac4067 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -47,7 +47,8 @@ struct SummingSortedAlgorithm::AggregateDescription void init(const char * function_name, const DataTypes & argument_types) { - function = AggregateFunctionFactory::instance().get(function_name, argument_types); + AggregateFunctionProperties properties; + function = AggregateFunctionFactory::instance().get(function_name, argument_types, {}, properties); add_function = function->getAddressOfAddFunction(); state.reset(function->sizeOfData(), function->alignOfData()); } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index e08ea1739a5..13cce2b0536 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -117,8 +117,9 @@ static void appendGraphitePattern( aggregate_function_name, params_row, "GraphiteMergeTree storage initialization"); /// TODO Not only Float64 - pattern.function = AggregateFunctionFactory::instance().get(aggregate_function_name, {std::make_shared()}, - params_row); + AggregateFunctionProperties properties; + pattern.function = AggregateFunctionFactory::instance().get( + aggregate_function_name, {std::make_shared()}, params_row, properties); } else if (startsWith(key, "retention")) { diff --git a/src/TableFunctions/TableFunctionFactory.cpp b/src/TableFunctions/TableFunctionFactory.cpp index 1b34c1a1e6f..bc139edfb73 100644 --- a/src/TableFunctions/TableFunctionFactory.cpp +++ b/src/TableFunctions/TableFunctionFactory.cpp @@ -15,7 +15,7 @@ namespace ErrorCodes } -void TableFunctionFactory::registerFunction(const std::string & name, Creator creator, CaseSensitiveness case_sensitiveness) +void TableFunctionFactory::registerFunction(const std::string & name, Value creator, CaseSensitiveness case_sensitiveness) { if (!table_functions.emplace(name, creator).second) throw Exception("TableFunctionFactory: the table function name '" + name + "' is not unique", diff --git a/src/TableFunctions/TableFunctionFactory.h b/src/TableFunctions/TableFunctionFactory.h index cd87fa9c7f0..6d0302a64ff 100644 --- a/src/TableFunctions/TableFunctionFactory.h +++ b/src/TableFunctions/TableFunctionFactory.h @@ -24,12 +24,11 @@ using TableFunctionCreator = std::function; class TableFunctionFactory final: private boost::noncopyable, public IFactoryWithAliases { public: - static TableFunctionFactory & instance(); /// Register a function by its name. /// No locking, you must register all functions before usage of get. - void registerFunction(const std::string & name, Creator creator, CaseSensitiveness case_sensitiveness = CaseSensitive); + void registerFunction(const std::string & name, Value creator, CaseSensitiveness case_sensitiveness = CaseSensitive); template void registerFunction(CaseSensitiveness case_sensitiveness = CaseSensitive) @@ -50,11 +49,11 @@ public: bool isTableFunctionName(const std::string & name) const; private: - using TableFunctions = std::unordered_map; + using TableFunctions = std::unordered_map; - const TableFunctions & getCreatorMap() const override { return table_functions; } + const TableFunctions & getMap() const override { return table_functions; } - const TableFunctions & getCaseInsensitiveCreatorMap() const override { return case_insensitive_table_functions; } + const TableFunctions & getCaseInsensitiveMap() const override { return case_insensitive_table_functions; } String getFactoryName() const override { return "TableFunctionFactory"; } diff --git a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference index f8b77704aa3..76b82419556 100644 --- a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference +++ b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference @@ -5,5 +5,5 @@ 5 5 0 -\N -\N +0 +0 diff --git a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql index 2d9b5ef54aa..9787ee2bd70 100644 --- a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql +++ b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql @@ -7,6 +7,5 @@ SELECT uniqExact(number >= 5 ? number : NULL) FROM numbers(10); SELECT count(DISTINCT number >= 5 ? number : NULL) FROM numbers(10); SELECT count(NULL); --- These two returns NULL for now, but we want to change them to return 0. SELECT uniq(NULL); SELECT count(DISTINCT NULL); diff --git a/utils/convert-month-partitioned-parts/CMakeLists.txt b/utils/convert-month-partitioned-parts/CMakeLists.txt index 14853590c76..ea6429a0610 100644 --- a/utils/convert-month-partitioned-parts/CMakeLists.txt +++ b/utils/convert-month-partitioned-parts/CMakeLists.txt @@ -1,2 +1,2 @@ add_executable (convert-month-partitioned-parts main.cpp) -target_link_libraries(convert-month-partitioned-parts PRIVATE dbms clickhouse_parsers boost::program_options) +target_link_libraries(convert-month-partitioned-parts PRIVATE clickhouse_aggregate_functions dbms clickhouse_parsers boost::program_options) diff --git a/utils/zookeeper-adjust-block-numbers-to-parts/CMakeLists.txt b/utils/zookeeper-adjust-block-numbers-to-parts/CMakeLists.txt index 08907e1c5b9..882c510ea1c 100644 --- a/utils/zookeeper-adjust-block-numbers-to-parts/CMakeLists.txt +++ b/utils/zookeeper-adjust-block-numbers-to-parts/CMakeLists.txt @@ -1,3 +1,3 @@ add_executable (zookeeper-adjust-block-numbers-to-parts main.cpp ${SRCS}) target_compile_options(zookeeper-adjust-block-numbers-to-parts PRIVATE -Wno-format) -target_link_libraries (zookeeper-adjust-block-numbers-to-parts PRIVATE dbms clickhouse_common_zookeeper boost::program_options) +target_link_libraries (zookeeper-adjust-block-numbers-to-parts PRIVATE clickhouse_aggregate_functions dbms clickhouse_common_zookeeper boost::program_options) From 217d05443a654bb35b760c2d144aa3516d30fd97 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 20:41:45 +0300 Subject: [PATCH 177/329] Remove old non-automated test --- src/Client/tests/CMakeLists.txt | 2 -- src/Client/tests/test_connect.cpp | 59 ------------------------------- 2 files changed, 61 deletions(-) delete mode 100644 src/Client/tests/test_connect.cpp diff --git a/src/Client/tests/CMakeLists.txt b/src/Client/tests/CMakeLists.txt index d952c006bb5..e69de29bb2d 100644 --- a/src/Client/tests/CMakeLists.txt +++ b/src/Client/tests/CMakeLists.txt @@ -1,2 +0,0 @@ -add_executable(test-connect test_connect.cpp) -target_link_libraries (test-connect PRIVATE dbms) diff --git a/src/Client/tests/test_connect.cpp b/src/Client/tests/test_connect.cpp deleted file mode 100644 index 50075cc24a6..00000000000 --- a/src/Client/tests/test_connect.cpp +++ /dev/null @@ -1,59 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include - - -/** In a loop it connects to the server and immediately breaks the connection. - * Using the SO_LINGER option, we ensure that the connection is terminated by sending a RST packet (not FIN). - * This behavior causes a bug in the TCPServer implementation in the Poco library. - */ -int main(int argc, char ** argv) -try -{ - for (size_t i = 0, num_iters = argc >= 2 ? DB::parse(argv[1]) : 1; i < num_iters; ++i) - { - std::cerr << "."; - - Poco::Net::SocketAddress address("localhost", 9000); - - int fd = socket(PF_INET, SOCK_STREAM, IPPROTO_IP); - - if (fd < 0) - DB::throwFromErrno("Cannot create socket", 0); - - linger linger_value; - linger_value.l_onoff = 1; - linger_value.l_linger = 0; - - if (0 != setsockopt(fd, SOL_SOCKET, SO_LINGER, &linger_value, sizeof(linger_value))) - DB::throwFromErrno("Cannot set linger", 0); - - try - { - int res = connect(fd, address.addr(), address.length()); - - if (res != 0 && errno != EINPROGRESS && errno != EWOULDBLOCK) - { - close(fd); - DB::throwFromErrno("Cannot connect", 0); - } - - close(fd); - } - catch (const Poco::Exception & e) - { - std::cerr << e.displayText() << "\n"; - } - } - - std::cerr << "\n"; -} -catch (const Poco::Exception & e) -{ - std::cerr << e.displayText() << "\n"; -} From fcd23d02eec3fd7029d3090f31a117f75bbe3be1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 20:57:50 +0300 Subject: [PATCH 178/329] Fix timeout in sql_fuzzy test --- tests/queries/0_stateless/00746_sql_fuzzy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00746_sql_fuzzy.sh b/tests/queries/0_stateless/00746_sql_fuzzy.sh index f417f0146c2..539d0659d36 100755 --- a/tests/queries/0_stateless/00746_sql_fuzzy.sh +++ b/tests/queries/0_stateless/00746_sql_fuzzy.sh @@ -13,7 +13,7 @@ $CLICKHOUSE_CLIENT -q "select name from system.table_functions format TSV;" > $S # if you want long run use: env SQL_FUZZY_RUNS=100000 clickhouse-test sql_fuzzy for SQL_FUZZY_RUN in $(seq ${SQL_FUZZY_RUNS:=10}); do - env SQL_FUZZY_RUN=$SQL_FUZZY_RUN $CURDIR/00746_sql_fuzzy.pl | $CLICKHOUSE_CLIENT --max_execution_time 10 -n --ignore-error >/dev/null 2>&1 + env SQL_FUZZY_RUN=$SQL_FUZZY_RUN $CURDIR/00746_sql_fuzzy.pl | $CLICKHOUSE_CLIENT --format Null --max_execution_time 10 -n --ignore-error >/dev/null 2>&1 if [[ `$CLICKHOUSE_CLIENT -q "SELECT 'Still alive'"` != 'Still alive' ]]; then break fi From 5d891f6c878562329c8a488debd704317c63b90d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 21:13:39 +0300 Subject: [PATCH 179/329] Fix tests --- .../0_stateless/01277_convert_field_to_type_logical_error.sql | 2 +- .../0_stateless/01311_comparison_with_constant_string.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01277_convert_field_to_type_logical_error.sql b/tests/queries/0_stateless/01277_convert_field_to_type_logical_error.sql index 05295575cf1..4712c124237 100644 --- a/tests/queries/0_stateless/01277_convert_field_to_type_logical_error.sql +++ b/tests/queries/0_stateless/01277_convert_field_to_type_logical_error.sql @@ -1 +1 @@ -SELECT -2487, globalNullIn(toIntervalMinute(-88074), 'qEkek..'), [-27.537293]; -- { serverError 70 } +SELECT -2487, globalNullIn(toIntervalMinute(-88074), 'qEkek..'), [-27.537293]; -- { serverError 53 } diff --git a/tests/queries/0_stateless/01311_comparison_with_constant_string.sql b/tests/queries/0_stateless/01311_comparison_with_constant_string.sql index 6ca736ba146..d6641a50c45 100644 --- a/tests/queries/0_stateless/01311_comparison_with_constant_string.sql +++ b/tests/queries/0_stateless/01311_comparison_with_constant_string.sql @@ -11,7 +11,7 @@ SELECT 1 IN (1.23, '2', 2); SELECT '---'; -- it should work but it doesn't. -SELECT 1 = '1.0'; -- { serverError 131 } +SELECT 1 = '1.0'; -- { serverError 53 } SELECT '---'; SELECT 1 = '257'; From db0fc6c9a661ed1f1443e95f1f31252a12bf8244 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 20:48:10 +0300 Subject: [PATCH 180/329] Fix tests --- src/DataTypes/DataTypeDateTime64.h | 2 ++ src/Interpreters/convertFieldToType.cpp | 8 +++++++- tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h index b575e9d81c1..249da255eb0 100644 --- a/src/DataTypes/DataTypeDateTime64.h +++ b/src/DataTypes/DataTypeDateTime64.h @@ -45,6 +45,8 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; bool equals(const IDataType & rhs) const override; + + bool canBePromoted() const override { return false; } }; /** Tansform-type wrapper for DateTime64, applies given Transform to DateTime64 value or only to a whole part of it. diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 21cf9422c32..d46573d0461 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -180,7 +180,13 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID return src; } - // TODO (vnemkov): extra cases for DateTime64: converting from integer, converting from Decimal + if (which_type.isDateTime64() && src.getType() == Field::Types::Decimal64) + { + /// Already in needed type. + return src; + } + + /// TODO Conversion from integers to DateTime64 } else if (which_type.isStringOrFixedString()) { diff --git a/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql b/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql index 7848b4aaf24..4ffcf3be3c9 100644 --- a/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql +++ b/tests/queries/0_stateless/01268_DateTime64_in_WHERE.sql @@ -5,8 +5,8 @@ WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT materiali WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT * WHERE DT64 = materialize(S); -- {serverError 43} WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT * WHERE materialize(S) = DT64; -- {serverError 43} -SELECT * WHERE toDateTime64(123.345, 3) == 'ABCD'; -- {serverError 131} -- invalid DateTime64 string -SELECT * WHERE toDateTime64(123.345, 3) == '2020-02-05 14:34:12.33333333333333333333333333333333333333333333333333333333'; -- {serverError 131} -- invalid string length +SELECT * WHERE toDateTime64(123.345, 3) == 'ABCD'; -- {serverError 53} -- invalid DateTime64 string +SELECT * WHERE toDateTime64(123.345, 3) == '2020-02-05 14:34:12.33333333333333333333333333333333333333333333333333333333'; -- {serverError 53} -- invalid string length SELECT 'in SELECT'; WITH '2020-02-05 14:34:12.333' as S, toDateTime64(S, 3) as DT64 SELECT DT64 = S; From f6c52fe1c225cc53a3e184d6a8e9433733d2b59c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 03:16:01 +0300 Subject: [PATCH 181/329] Allow comparison with String in index analysis; simplify code #11630 --- src/Common/FieldVisitors.h | 303 ++++++++---------------- src/Storages/MergeTree/KeyCondition.cpp | 4 +- 2 files changed, 98 insertions(+), 209 deletions(-) diff --git a/src/Common/FieldVisitors.h b/src/Common/FieldVisitors.h index 90f80974ab1..257994a6bd2 100644 --- a/src/Common/FieldVisitors.h +++ b/src/Common/FieldVisitors.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include class SipHash; @@ -184,232 +186,119 @@ template <> constexpr bool isDecimalField>() { return t class FieldVisitorAccurateEquals : public StaticVisitor { public: - bool operator() (const UInt64 &, const Null &) const { return false; } - bool operator() (const UInt64 & l, const UInt64 & r) const { return l == r; } - bool operator() (const UInt64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Int64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const UInt64 & l, const Float64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const UInt64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - bool operator() (const Int64 &, const Null &) const { return false; } - bool operator() (const Int64 & l, const UInt64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const Int64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Int64 & r) const { return l == r; } - bool operator() (const Int64 & l, const Float64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const Int64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - bool operator() (const Float64 &, const Null &) const { return false; } - bool operator() (const Float64 & l, const UInt64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const Float64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Int64 & r) const { return accurate::equalsOp(l, r); } - bool operator() (const Float64 & l, const Float64 & r) const { return l == r; } - bool operator() (const Float64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - template - bool operator() (const Null &, const T &) const - { - return std::is_same_v; - } - - template - bool operator() (const String & l, const T & r) const - { - if constexpr (std::is_same_v) - return l == r; - if constexpr (std::is_same_v) - return stringToUUID(l) == r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const UInt128 & l, const T & r) const - { - if constexpr (std::is_same_v) - return l == r; - if constexpr (std::is_same_v) - return l == stringToUUID(r); - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const Array & l, const T & r) const - { - if constexpr (std::is_same_v) - return l == r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const Tuple & l, const T & r) const - { - if constexpr (std::is_same_v) - return l == r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - template - bool operator() (const DecimalField & l, const U & r) const + bool operator() (const T & l, const U & r) const { - if constexpr (isDecimalField()) - return l == r; - if constexpr (std::is_same_v || std::is_same_v) - return l == DecimalField(r, 0); - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } + if constexpr (std::is_same_v || std::is_same_v) + return std::is_same_v; + else + { + if constexpr (std::is_same_v) + return l == r; - template bool operator() (const UInt64 & l, const DecimalField & r) const { return DecimalField(l, 0) == r; } - template bool operator() (const Int64 & l, const DecimalField & r) const { return DecimalField(l, 0) == r; } - template bool operator() (const Float64 & l, const DecimalField & r) const { return cantCompare(l, r); } + if constexpr (std::is_arithmetic_v && std::is_arithmetic_v) + return accurate::equalsOp(l, r); - template - bool operator() (const AggregateFunctionStateData & l, const T & r) const - { - if constexpr (std::is_same_v) - return l == r; - return cantCompare(l, r); - } + if constexpr (isDecimalField() && isDecimalField()) + return l == r; + + if constexpr (isDecimalField() && std::is_arithmetic_v) + return l == DecimalField(r, 0); + + if constexpr (std::is_arithmetic_v && isDecimalField()) + return DecimalField(l, 0) == r; + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return stringToUUID(l) == r; + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(l); + T parsed; + readText(parsed, in); + return operator()(parsed, r); + } + } + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return l == stringToUUID(r); + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(r); + T parsed; + readText(parsed, in); + return operator()(l, parsed); + } + } + } -private: - template - bool cantCompare(const T &, const U &) const - { - if constexpr (std::is_same_v) - return false; throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()), - ErrorCodes::BAD_TYPE_OF_FIELD); + ErrorCodes::BAD_TYPE_OF_FIELD); } }; + class FieldVisitorAccurateLess : public StaticVisitor { public: - bool operator() (const UInt64 &, const Null &) const { return false; } - bool operator() (const UInt64 & l, const UInt64 & r) const { return l < r; } - bool operator() (const UInt64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Int64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const UInt64 & l, const Float64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const UInt64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const UInt64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - bool operator() (const Int64 &, const Null &) const { return false; } - bool operator() (const Int64 & l, const UInt64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const Int64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Int64 & r) const { return l < r; } - bool operator() (const Int64 & l, const Float64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const Int64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - bool operator() (const Float64 &, const Null &) const { return false; } - bool operator() (const Float64 & l, const UInt64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const Float64 & l, const UInt128 & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Int64 & r) const { return accurate::lessOp(l, r); } - bool operator() (const Float64 & l, const Float64 & r) const { return l < r; } - bool operator() (const Float64 & l, const String & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Array & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Tuple & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - - template - bool operator() (const Null &, const T &) const - { - return !std::is_same_v; - } - - template - bool operator() (const String & l, const T & r) const - { - if constexpr (std::is_same_v) - return l < r; - if constexpr (std::is_same_v) - return stringToUUID(l) < r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const UInt128 & l, const T & r) const - { - if constexpr (std::is_same_v) - return l < r; - if constexpr (std::is_same_v) - return l < stringToUUID(r); - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const Array & l, const T & r) const - { - if constexpr (std::is_same_v) - return l < r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - - template - bool operator() (const Tuple & l, const T & r) const - { - if constexpr (std::is_same_v) - return l < r; - if constexpr (std::is_same_v) - return false; - return cantCompare(l, r); - } - template - bool operator() (const DecimalField & l, const U & r) const + bool operator() (const T & l, const U & r) const { - if constexpr (isDecimalField()) - return l < r; - if constexpr (std::is_same_v || std::is_same_v) - return l < DecimalField(r, 0); - if constexpr (std::is_same_v) + if constexpr (std::is_same_v || std::is_same_v) return false; - return cantCompare(l, r); - } + else + { + if constexpr (std::is_same_v) + return l < r; - template bool operator() (const UInt64 & l, const DecimalField & r) const { return DecimalField(l, 0) < r; } - template bool operator() (const Int64 & l, const DecimalField & r) const { return DecimalField(l, 0) < r; } - template bool operator() (const Float64 &, const DecimalField &) const { return false; } + if constexpr (std::is_arithmetic_v && std::is_arithmetic_v) + return accurate::lessOp(l, r); - template - bool operator() (const AggregateFunctionStateData & l, const T & r) const - { - return cantCompare(l, r); - } + if constexpr (isDecimalField() && isDecimalField()) + return l < r; + + if constexpr (isDecimalField() && std::is_arithmetic_v) + return l < DecimalField(r, 0); + + if constexpr (std::is_arithmetic_v && isDecimalField()) + return DecimalField(l, 0) < r; + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return stringToUUID(l) < r; + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(l); + T parsed; + readText(parsed, in); + return operator()(parsed, r); + } + } + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return l < stringToUUID(r); + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(r); + T parsed; + readText(parsed, in); + return operator()(l, parsed); + } + } + } -private: - template - bool cantCompare(const T &, const U &) const - { throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()), - ErrorCodes::BAD_TYPE_OF_FIELD); + ErrorCodes::BAD_TYPE_OF_FIELD); } }; diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index dad73b6a003..7265e818b51 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -826,8 +826,8 @@ bool KeyCondition::tryParseAtomFromAST(const ASTPtr & node, const Context & cont } bool cast_not_needed = - is_set_const /// Set args are already casted inside Set::createFromAST - || (isNativeNumber(key_expr_type) && isNativeNumber(const_type)); /// Numbers are accurately compared without cast. + is_set_const /// Set args are already casted inside Set::createFromAST + || (isNativeNumber(key_expr_type) && isNativeNumber(const_type)); /// Numbers are accurately compared without cast. if (!cast_not_needed) castValueToType(key_expr_type, const_value, const_type, node); From 3aedef99ce8395dd6e2c947dce4e999e95fa9bc6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 03:24:55 +0300 Subject: [PATCH 182/329] Added a test --- ...onstant_string_in_index_analysis.reference | 12 +++++++ ...with_constant_string_in_index_analysis.sql | 32 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.reference create mode 100644 tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql diff --git a/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.reference b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.reference new file mode 100644 index 00000000000..ee98bdf033b --- /dev/null +++ b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.reference @@ -0,0 +1,12 @@ +1 +999999 +100000 +899999 +100001 +900000 +1 +999999 +100000 +899999 +100001 +900000 diff --git a/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql new file mode 100644 index 00000000000..e37f647e81f --- /dev/null +++ b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS test; +CREATE TABLE test (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 1000; +INSERT INTO test SELECT * FROM numbers(1000000); +OPTIMIZE TABLE test; + +SET max_rows_to_read = 2000; +SELECT count() FROM test WHERE x = 100000; +SET max_rows_to_read = 1000000; +SELECT count() FROM test WHERE x != 100000; +SET max_rows_to_read = 101000; +SELECT count() FROM test WHERE x < 100000; +SET max_rows_to_read = 900000; +SELECT count() FROM test WHERE x > 100000; +SET max_rows_to_read = 101000; +SELECT count() FROM test WHERE x <= 100000; +SET max_rows_to_read = 901000; +SELECT count() FROM test WHERE x >= 100000; + +SET max_rows_to_read = 2000; +SELECT count() FROM test WHERE x = '100000'; +SET max_rows_to_read = 1000000; +SELECT count() FROM test WHERE x != '100000'; +SET max_rows_to_read = 101000; +SELECT count() FROM test WHERE x < '100000'; +SET max_rows_to_read = 900000; +SELECT count() FROM test WHERE x > '100000'; +SET max_rows_to_read = 101000; +SELECT count() FROM test WHERE x <= '100000'; +SET max_rows_to_read = 901000; +SELECT count() FROM test WHERE x >= '100000'; + +DROP TABLE test; From 067cf4cc403e512d09a9dbcc4e0178d5b29278d6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 07:52:28 +0300 Subject: [PATCH 183/329] Fix gcc build --- src/Common/Arena.h | 2 +- src/Common/ArenaWithFreeLists.h | 2 +- src/Core/Defines.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Common/Arena.h b/src/Common/Arena.h index d203a92d4a3..aaf71cac525 100644 --- a/src/Common/Arena.h +++ b/src/Common/Arena.h @@ -4,7 +4,7 @@ #include #include #include -#if __has_include() +#if __has_include() && defined(ADDRESS_SANITIZER) # include #endif #include diff --git a/src/Common/ArenaWithFreeLists.h b/src/Common/ArenaWithFreeLists.h index 6092f03ce19..3ae727fdaa5 100644 --- a/src/Common/ArenaWithFreeLists.h +++ b/src/Common/ArenaWithFreeLists.h @@ -1,6 +1,6 @@ #pragma once -#if __has_include() +#if __has_include() && defined(ADDRESS_SANITIZER) # include #endif #include diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 13070c565b4..8b26f486c9d 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -87,7 +87,7 @@ #define DBMS_DISTRIBUTED_SIGNATURE_HEADER 0xCAFEDACEull #define DBMS_DISTRIBUTED_SIGNATURE_HEADER_OLD_FORMAT 0xCAFECABEull -#if !__has_include() +#if !__has_include() || !defined(ADDRESS_SANITIZER) # define ASAN_UNPOISON_MEMORY_REGION(a, b) # define ASAN_POISON_MEMORY_REGION(a, b) #endif From 8dac30ae955a1ef0b78826d8d7b06594e583263d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 21:42:10 +0300 Subject: [PATCH 184/329] Split file for better build times --- src/Common/FieldVisitors.h | 128 ----------------- src/Common/FieldVisitorsAccurateComparison.h | 142 +++++++++++++++++++ src/Functions/array/arrayIndex.h | 2 +- src/Interpreters/FillingRow.cpp | 2 + src/Interpreters/FillingRow.h | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Storages/MergeTree/KeyCondition.cpp | 2 +- 7 files changed, 148 insertions(+), 132 deletions(-) create mode 100644 src/Common/FieldVisitorsAccurateComparison.h diff --git a/src/Common/FieldVisitors.h b/src/Common/FieldVisitors.h index 257994a6bd2..ddeddb8fbf6 100644 --- a/src/Common/FieldVisitors.h +++ b/src/Common/FieldVisitors.h @@ -1,10 +1,7 @@ #pragma once #include -#include #include -#include -#include class SipHash; @@ -16,7 +13,6 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_CONVERT_TYPE; - extern const int BAD_TYPE_OF_FIELD; extern const int LOGICAL_ERROR; } @@ -179,130 +175,6 @@ template <> constexpr bool isDecimalField>() { return tr template <> constexpr bool isDecimalField>() { return true; } -/** More precise comparison, used for index. - * Differs from Field::operator< and Field::operator== in that it also compares values of different types. - * Comparison rules are same as in FunctionsComparison (to be consistent with expression evaluation in query). - */ -class FieldVisitorAccurateEquals : public StaticVisitor -{ -public: - template - bool operator() (const T & l, const U & r) const - { - if constexpr (std::is_same_v || std::is_same_v) - return std::is_same_v; - else - { - if constexpr (std::is_same_v) - return l == r; - - if constexpr (std::is_arithmetic_v && std::is_arithmetic_v) - return accurate::equalsOp(l, r); - - if constexpr (isDecimalField() && isDecimalField()) - return l == r; - - if constexpr (isDecimalField() && std::is_arithmetic_v) - return l == DecimalField(r, 0); - - if constexpr (std::is_arithmetic_v && isDecimalField()) - return DecimalField(l, 0) == r; - - if constexpr (std::is_same_v) - { - if constexpr (std::is_same_v) - return stringToUUID(l) == r; - - if constexpr (std::is_arithmetic_v) - { - ReadBufferFromString in(l); - T parsed; - readText(parsed, in); - return operator()(parsed, r); - } - } - - if constexpr (std::is_same_v) - { - if constexpr (std::is_same_v) - return l == stringToUUID(r); - - if constexpr (std::is_arithmetic_v) - { - ReadBufferFromString in(r); - T parsed; - readText(parsed, in); - return operator()(l, parsed); - } - } - } - - throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()), - ErrorCodes::BAD_TYPE_OF_FIELD); - } -}; - - -class FieldVisitorAccurateLess : public StaticVisitor -{ -public: - template - bool operator() (const T & l, const U & r) const - { - if constexpr (std::is_same_v || std::is_same_v) - return false; - else - { - if constexpr (std::is_same_v) - return l < r; - - if constexpr (std::is_arithmetic_v && std::is_arithmetic_v) - return accurate::lessOp(l, r); - - if constexpr (isDecimalField() && isDecimalField()) - return l < r; - - if constexpr (isDecimalField() && std::is_arithmetic_v) - return l < DecimalField(r, 0); - - if constexpr (std::is_arithmetic_v && isDecimalField()) - return DecimalField(l, 0) < r; - - if constexpr (std::is_same_v) - { - if constexpr (std::is_same_v) - return stringToUUID(l) < r; - - if constexpr (std::is_arithmetic_v) - { - ReadBufferFromString in(l); - T parsed; - readText(parsed, in); - return operator()(parsed, r); - } - } - - if constexpr (std::is_same_v) - { - if constexpr (std::is_same_v) - return l < stringToUUID(r); - - if constexpr (std::is_arithmetic_v) - { - ReadBufferFromString in(r); - T parsed; - readText(parsed, in); - return operator()(l, parsed); - } - } - } - - throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()), - ErrorCodes::BAD_TYPE_OF_FIELD); - } -}; - - /** Implements `+=` operation. * Returns false if the result is zero. */ diff --git a/src/Common/FieldVisitorsAccurateComparison.h b/src/Common/FieldVisitorsAccurateComparison.h new file mode 100644 index 00000000000..91fa4bf28de --- /dev/null +++ b/src/Common/FieldVisitorsAccurateComparison.h @@ -0,0 +1,142 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_TYPE_OF_FIELD; +} + +/** More precise comparison, used for index. + * Differs from Field::operator< and Field::operator== in that it also compares values of different types. + * Comparison rules are same as in FunctionsComparison (to be consistent with expression evaluation in query). + */ +class FieldVisitorAccurateEquals : public StaticVisitor +{ +public: + template + bool operator() (const T & l, const U & r) const + { + if constexpr (std::is_same_v || std::is_same_v) + return std::is_same_v; + else + { + if constexpr (std::is_same_v) + return l == r; + + if constexpr (std::is_arithmetic_v && std::is_arithmetic_v) + return accurate::equalsOp(l, r); + + if constexpr (isDecimalField() && isDecimalField()) + return l == r; + + if constexpr (isDecimalField() && std::is_arithmetic_v) + return l == DecimalField(r, 0); + + if constexpr (std::is_arithmetic_v && isDecimalField()) + return DecimalField(l, 0) == r; + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return stringToUUID(l) == r; + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(l); + T parsed; + readText(parsed, in); + return operator()(parsed, r); + } + } + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return l == stringToUUID(r); + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(r); + T parsed; + readText(parsed, in); + return operator()(l, parsed); + } + } + } + + throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()), + ErrorCodes::BAD_TYPE_OF_FIELD); + } +}; + + +class FieldVisitorAccurateLess : public StaticVisitor +{ +public: + template + bool operator() (const T & l, const U & r) const + { + if constexpr (std::is_same_v || std::is_same_v) + return false; + else + { + if constexpr (std::is_same_v) + return l < r; + + if constexpr (std::is_arithmetic_v && std::is_arithmetic_v) + return accurate::lessOp(l, r); + + if constexpr (isDecimalField() && isDecimalField()) + return l < r; + + if constexpr (isDecimalField() && std::is_arithmetic_v) + return l < DecimalField(r, 0); + + if constexpr (std::is_arithmetic_v && isDecimalField()) + return DecimalField(l, 0) < r; + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return stringToUUID(l) < r; + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(l); + T parsed; + readText(parsed, in); + return operator()(parsed, r); + } + } + + if constexpr (std::is_same_v) + { + if constexpr (std::is_same_v) + return l < stringToUUID(r); + + if constexpr (std::is_arithmetic_v) + { + ReadBufferFromString in(r); + T parsed; + readText(parsed, in); + return operator()(l, parsed); + } + } + } + + throw Exception("Cannot compare " + demangle(typeid(T).name()) + " with " + demangle(typeid(U).name()), + ErrorCodes::BAD_TYPE_OF_FIELD); + } +}; + +} diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index fab1332cbda..50214ee790f 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Interpreters/FillingRow.cpp b/src/Interpreters/FillingRow.cpp index dc48b5347c4..7e32d9514a6 100644 --- a/src/Interpreters/FillingRow.cpp +++ b/src/Interpreters/FillingRow.cpp @@ -1,4 +1,6 @@ #include +#include + namespace DB { diff --git a/src/Interpreters/FillingRow.h b/src/Interpreters/FillingRow.h index 1753508e139..0e1d60d0d7a 100644 --- a/src/Interpreters/FillingRow.h +++ b/src/Interpreters/FillingRow.h @@ -1,7 +1,7 @@ #pragma once #include #include -#include + namespace DB { diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index f9072e6176a..dc32371b6c1 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -37,7 +37,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 7265e818b51..281f8511a59 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include From 3663e2a47e1ac04ccd7aa0dd3ce41b8685a2de1a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 22:11:32 +0300 Subject: [PATCH 185/329] Fix syntax hilite in CREATE USER query --- src/Parsers/ASTCreateUserQuery.cpp | 3 ++- .../0_stateless/01316_create_user_syntax_hilite.reference | 1 + .../0_stateless/01316_create_user_syntax_hilite.sh | 8 ++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01316_create_user_syntax_hilite.reference create mode 100755 tests/queries/0_stateless/01316_create_user_syntax_hilite.sh diff --git a/src/Parsers/ASTCreateUserQuery.cpp b/src/Parsers/ASTCreateUserQuery.cpp index e5c1178285b..1d61303860a 100644 --- a/src/Parsers/ASTCreateUserQuery.cpp +++ b/src/Parsers/ASTCreateUserQuery.cpp @@ -65,7 +65,8 @@ namespace settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << authentication_type_name << (settings.hilite ? IAST::hilite_none : ""); if (password) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " BY " << quoteString(*password); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " BY " << (settings.hilite ? IAST::hilite_none : "") + << quoteString(*password); } diff --git a/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference b/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference new file mode 100644 index 00000000000..ed7daeb3609 --- /dev/null +++ b/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference @@ -0,0 +1 @@ +CREATE USER user IDENTIFIED WITH plaintext_password BY 'hello' diff --git a/tests/queries/0_stateless/01316_create_user_syntax_hilite.sh b/tests/queries/0_stateless/01316_create_user_syntax_hilite.sh new file mode 100755 index 00000000000..1031a96363c --- /dev/null +++ b/tests/queries/0_stateless/01316_create_user_syntax_hilite.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_FORMAT --hilite <<< "CREATE USER user IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'hello'" From 07ba7ffea52dbc0719cd7eccea77079125e39ebd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 22:23:05 +0300 Subject: [PATCH 186/329] Clear password from command line #11624 --- programs/benchmark/Benchmark.cpp | 5 ++++- programs/client/Client.cpp | 2 ++ src/Common/clearPasswordFromCommandLine.cpp | 18 ++++++++++++++++++ src/Common/clearPasswordFromCommandLine.h | 6 ++++++ src/Common/ya.make | 1 + 5 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 src/Common/clearPasswordFromCommandLine.cpp create mode 100644 src/Common/clearPasswordFromCommandLine.h diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index e17320b39ea..bb814f474e3 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -539,7 +540,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ("password", value()->default_value(""), "") ("database", value()->default_value("default"), "") ("stacktrace", "print stack traces of exceptions") - ("confidence", value()->default_value(5), "set the level of confidence for T-test [0=80%, 1=90%, 2=95%, 3=98%, 4=99%, 5=99.5%(default)") + ("confidence", value()->default_value(5), "set the level of confidence for T-test [0=80%, 1=90%, 2=95%, 3=98%, 4=99%, 5=99.5%(default)") ("query_id", value()->default_value(""), "") ; @@ -550,6 +551,8 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); boost::program_options::notify(options); + clearPasswordFromCommandLine(argc, argv); + if (options.count("help")) { std::cout << "Usage: " << argv[0] << " [options] < queries.txt\n"; diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 7808120d09e..63467c1129d 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -2006,6 +2007,7 @@ public: argsToConfig(common_arguments, config(), 100); + clearPasswordFromCommandLine(argc, argv); } }; diff --git a/src/Common/clearPasswordFromCommandLine.cpp b/src/Common/clearPasswordFromCommandLine.cpp new file mode 100644 index 00000000000..0ff56e25c3f --- /dev/null +++ b/src/Common/clearPasswordFromCommandLine.cpp @@ -0,0 +1,18 @@ +#include +#include "clearPasswordFromCommandLine.h" + +void clearPasswordFromCommandLine(int argc, char ** argv) +{ + for (int arg = 1; arg < argc; ++arg) + { + if (arg + 1 < argc && 0 == strcmp(argv[arg], "--password")) + { + ++arg; + memset(argv[arg], 0, strlen(argv[arg])); + } + else if (0 == strncmp(argv[arg], "--password=", strlen("--password="))) + { + memset(argv[arg] + strlen("--password="), 0, strlen(argv[arg]) - strlen("--password=")); + } + } +} diff --git a/src/Common/clearPasswordFromCommandLine.h b/src/Common/clearPasswordFromCommandLine.h new file mode 100644 index 00000000000..cf90fea1dc8 --- /dev/null +++ b/src/Common/clearPasswordFromCommandLine.h @@ -0,0 +1,6 @@ +#pragma once + +/** If there are --password=... or --password ... arguments in command line, replace their values with zero bytes. + * This is needed to prevent password exposure in 'ps' and similar tools. + */ +void clearPasswordFromCommandLine(int argc, char ** argv); diff --git a/src/Common/ya.make b/src/Common/ya.make index 83a419212bd..327089ff31d 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -30,6 +30,7 @@ SRCS( Config/configReadClient.cpp Config/ConfigReloader.cpp createHardLink.cpp + clearPasswordFromCommandLine.cpp CurrentMetrics.cpp CurrentThread.cpp DNSResolver.cpp From 22366471d03876402a46f1bd4e40602022562cf8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Jun 2020 22:31:45 +0300 Subject: [PATCH 187/329] Added a test --- ...1317_no_password_in_command_line.reference | 2 ++ .../01317_no_password_in_command_line.sh | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/queries/0_stateless/01317_no_password_in_command_line.reference create mode 100755 tests/queries/0_stateless/01317_no_password_in_command_line.sh diff --git a/tests/queries/0_stateless/01317_no_password_in_command_line.reference b/tests/queries/0_stateless/01317_no_password_in_command_line.reference new file mode 100644 index 00000000000..aa47d0d46d4 --- /dev/null +++ b/tests/queries/0_stateless/01317_no_password_in_command_line.reference @@ -0,0 +1,2 @@ +0 +0 diff --git a/tests/queries/0_stateless/01317_no_password_in_command_line.sh b/tests/queries/0_stateless/01317_no_password_in_command_line.sh new file mode 100755 index 00000000000..1a3ae88616a --- /dev/null +++ b/tests/queries/0_stateless/01317_no_password_in_command_line.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS user" +$CLICKHOUSE_CLIENT --query "CREATE USER user IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'hello'" + +# False positive result due to race condition with sleeps is Ok. + +$CLICKHOUSE_CLIENT --user user --password hello --query "SELECT sleep(1)" & +sleep 0.1 +ps auxw | grep -F -- '--password' | grep -F hello ||: +wait + +$CLICKHOUSE_CLIENT --user user --password=hello --query "SELECT sleep(1)" & +sleep 0.1 +ps auxw | grep -F -- '--password' | grep -F hello ||: +wait + +$CLICKHOUSE_CLIENT --query "DROP USER user" From c4f18d2896b8ed5721b34de68788b0b3ac300eb0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 00:39:22 +0300 Subject: [PATCH 188/329] Fix tests --- src/AggregateFunctions/AggregateFunctionFactory.cpp | 2 +- src/AggregateFunctions/AggregateFunctionNull.cpp | 2 -- tests/queries/0_stateless/00808_array_enumerate_segfault.sql | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index 7ff52fe0f70..c52b4d122dd 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -117,7 +117,7 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( /// The case when aggregate function should return NULL on NULL arguments. This case is handled in "get" method. if (!out_properties.returns_default_when_only_null - && std::any_of(argument_types.begin(), argument_types.end(), [](const auto & type) { return type->onlyNull(); })) + && std::any_of(argument_types.begin(), argument_types.end(), [](const auto & type) { return WhichDataType(type).isNothing(); })) { return nullptr; } diff --git a/src/AggregateFunctions/AggregateFunctionNull.cpp b/src/AggregateFunctions/AggregateFunctionNull.cpp index 85d960eae62..143c9b6246f 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.cpp +++ b/src/AggregateFunctions/AggregateFunctionNull.cpp @@ -57,8 +57,6 @@ public: if (has_null_types) { - std::cerr << properties.returns_default_when_only_null << "\n"; - /// Currently the only functions that returns not-NULL on all NULL arguments are count and uniq, and they returns UInt64. if (properties.returns_default_when_only_null) return std::make_shared(DataTypes{std::make_shared()}, params); diff --git a/tests/queries/0_stateless/00808_array_enumerate_segfault.sql b/tests/queries/0_stateless/00808_array_enumerate_segfault.sql index b492d3114f8..e5acba9cb57 100644 --- a/tests/queries/0_stateless/00808_array_enumerate_segfault.sql +++ b/tests/queries/0_stateless/00808_array_enumerate_segfault.sql @@ -1,4 +1,4 @@ SET send_logs_level = 'none'; SELECT arrayEnumerateUniq(anyHeavy([]), []); -SELECT arrayEnumerateDense([], [sequenceCount(NULL)]); -- { serverError 190 } +SELECT arrayEnumerateDense([], [sequenceCount(NULL)]); -- { serverError 42 } SELECT arrayEnumerateDense([STDDEV_SAMP(NULL, 910947.571364)], [NULL]); From 303d1ebdafbe81d3b61572dbb5670f85c62b885f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 01:17:06 +0300 Subject: [PATCH 189/329] Fix error; more tests --- src/AggregateFunctions/AggregateFunctionNull.cpp | 6 ++++-- .../01315_count_distinct_return_not_nullable.reference | 7 +++++++ .../01315_count_distinct_return_not_nullable.sql | 10 ++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionNull.cpp b/src/AggregateFunctions/AggregateFunctionNull.cpp index 143c9b6246f..b8fbad53350 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.cpp +++ b/src/AggregateFunctions/AggregateFunctionNull.cpp @@ -59,9 +59,11 @@ public: { /// Currently the only functions that returns not-NULL on all NULL arguments are count and uniq, and they returns UInt64. if (properties.returns_default_when_only_null) - return std::make_shared(DataTypes{std::make_shared()}, params); + return std::make_shared(DataTypes{ + std::make_shared()}, params); else - return std::make_shared(arguments, params); + return std::make_shared(DataTypes{ + std::make_shared(std::make_shared())}, params); } assert(nested_function); diff --git a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference index 76b82419556..b529a357af4 100644 --- a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference +++ b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.reference @@ -4,6 +4,13 @@ 5 5 5 +--- 0 0 0 +--- +\N +\N +\N +\N +\N diff --git a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql index 9787ee2bd70..932cd2f69f9 100644 --- a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql +++ b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql @@ -6,6 +6,16 @@ SELECT uniq(number >= 5 ? number : NULL) FROM numbers(10); SELECT uniqExact(number >= 5 ? number : NULL) FROM numbers(10); SELECT count(DISTINCT number >= 5 ? number : NULL) FROM numbers(10); +SELECT '---'; + SELECT count(NULL); SELECT uniq(NULL); SELECT count(DISTINCT NULL); + +SELECT '---'; + +SELECT avg(NULL); +SELECT sum(NULL); +SELECT corr(NULL, NULL); +SELECT corr(1, NULL); +SELECT corr(NULL, 1); From 6467302ad32b4dd6205542675d54fa89944d0ff1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 01:29:22 +0300 Subject: [PATCH 190/329] Fix gcc build --- src/Common/Arena.h | 2 +- src/Common/ArenaWithFreeLists.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/Arena.h b/src/Common/Arena.h index aaf71cac525..44a9b444ff2 100644 --- a/src/Common/Arena.h +++ b/src/Common/Arena.h @@ -4,10 +4,10 @@ #include #include #include +#include #if __has_include() && defined(ADDRESS_SANITIZER) # include #endif -#include #include #include #include diff --git a/src/Common/ArenaWithFreeLists.h b/src/Common/ArenaWithFreeLists.h index 3ae727fdaa5..1284c3586c0 100644 --- a/src/Common/ArenaWithFreeLists.h +++ b/src/Common/ArenaWithFreeLists.h @@ -1,9 +1,9 @@ #pragma once +#include #if __has_include() && defined(ADDRESS_SANITIZER) # include #endif -#include #include #include From ceaaf67d3fb6ae409a3515eac4ab0d63ae22303d Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 5 Jun 2020 23:44:10 +0300 Subject: [PATCH 191/329] Fix parsing CREATE SETTINGS PROFILE with WRITABLE keyword. --- src/Parsers/ParserSettingsProfileElement.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ParserSettingsProfileElement.cpp b/src/Parsers/ParserSettingsProfileElement.cpp index 37044e8ccbe..1dccae50cf5 100644 --- a/src/Parsers/ParserSettingsProfileElement.cpp +++ b/src/Parsers/ParserSettingsProfileElement.cpp @@ -87,7 +87,7 @@ namespace readonly = true; return true; } - else if (ParserKeyword{"READONLY"}.ignore(pos, expected)) + else if (ParserKeyword{"WRITABLE"}.ignore(pos, expected)) { readonly = false; return true; From ca2fb5932126175fed213a6f040fd34ff7b2d908 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 6 Jun 2020 05:25:27 +0300 Subject: [PATCH 192/329] Fix calculating full names of row policies. --- src/Access/RowPolicy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Access/RowPolicy.cpp b/src/Access/RowPolicy.cpp index 4249f351eae..acacaf01c6c 100644 --- a/src/Access/RowPolicy.cpp +++ b/src/Access/RowPolicy.cpp @@ -17,7 +17,7 @@ String RowPolicy::NameParts::getName() const name.reserve(database.length() + table_name.length() + short_name.length() + 6); name += backQuoteIfNeed(short_name); name += " ON "; - if (!name.empty()) + if (!database.empty()) { name += backQuoteIfNeed(database); name += '.'; From 3ffcb8e790434245cfeea7aceb9dbd8daf6a003b Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 15 Jun 2020 00:00:57 +0300 Subject: [PATCH 193/329] Fix casting values of settings while reading profiles from users.xml. --- src/Access/UsersConfigAccessStorage.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index f5f48a2390e..4d7d1b4cdfe 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -353,16 +353,17 @@ namespace for (const String & name : names) { SettingsProfileElement profile_element; - profile_element.setting_index = Settings::findIndexStrict(name); + size_t setting_index = Settings::findIndexStrict(name); + profile_element.setting_index = setting_index; Poco::Util::AbstractConfiguration::Keys constraint_types; String path_to_name = path_to_constraints + "." + name; config.keys(path_to_name, constraint_types); for (const String & constraint_type : constraint_types) { if (constraint_type == "min") - profile_element.min_value = config.getString(path_to_name + "." + constraint_type); + profile_element.min_value = Settings::valueToCorrespondingType(setting_index, config.getString(path_to_name + "." + constraint_type)); else if (constraint_type == "max") - profile_element.max_value = config.getString(path_to_name + "." + constraint_type); + profile_element.max_value = Settings::valueToCorrespondingType(setting_index, config.getString(path_to_name + "." + constraint_type)); else if (constraint_type == "readonly") profile_element.readonly = true; else @@ -402,8 +403,9 @@ namespace } SettingsProfileElement profile_element; - profile_element.setting_index = Settings::findIndexStrict(key); - profile_element.value = config.getString(profile_config + "." + key); + size_t setting_index = Settings::findIndexStrict(key); + profile_element.setting_index = setting_index; + profile_element.value = Settings::valueToCorrespondingType(setting_index, config.getString(profile_config + "." + key)); profile->elements.emplace_back(std::move(profile_element)); } From 1c438a133ea1db5efea9526dbafe5c9c4762a368 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 03:59:12 +0300 Subject: [PATCH 194/329] Leader election both backward and forward compatible --- src/Storages/MergeTree/LeaderElection.h | 68 ++++++++++++++----------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/src/Storages/MergeTree/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h index ef6b68bbe15..4d3a533d139 100644 --- a/src/Storages/MergeTree/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -33,10 +33,8 @@ namespace zkutil * to maintain compatibility when replicas with different versions work on the same cluster * (this is allowed for short time period during cluster update). * - * Replicas with old versions participate in leader election with ephemeral sequential nodes. - * If the node is first, then replica is the leader. - * Replicas with new versions creates persistent sequential nodes. - * If the first node is persistent, then all replicas with new versions become leaders. + * Replicas with new versions creates ephemeral sequential nodes with values like "replica_name (multiple leaders Ok)". + * If the first node belongs to a replica with new version, then all replicas with new versions become leaders. */ class LeaderElection { @@ -55,7 +53,7 @@ public: ZooKeeper & zookeeper_, LeadershipHandler handler_, const std::string & identifier_) - : pool(pool_), path(path_), zookeeper(zookeeper_), handler(handler_), identifier(identifier_) + : pool(pool_), path(path_), zookeeper(zookeeper_), handler(handler_), identifier(identifier_ + suffix) , log_name("LeaderElection (" + path + ")") , log(&Poco::Logger::get(log_name)) { @@ -74,19 +72,23 @@ public: ~LeaderElection() { - shutdown(); + releaseNode(); } private: + static inline constexpr auto suffix = " (multiple leaders Ok)"; DB::BackgroundSchedulePool & pool; DB::BackgroundSchedulePool::TaskHolder task; - const std::string path; + std::string path; ZooKeeper & zookeeper; LeadershipHandler handler; std::string identifier; std::string log_name; Poco::Logger * log; + EphemeralNodeHolderPtr node; + std::string node_name; + std::atomic shutdown_called {false}; CurrentMetrics::Increment metric_increment{CurrentMetrics::LeaderElection}; @@ -94,45 +96,52 @@ private: void createNode() { shutdown_called = false; + node = EphemeralNodeHolder::createSequential(path + "/leader_election-", zookeeper, identifier); - /// If there is at least one persistent node, we don't have to create another. - Strings children = zookeeper.getChildren(path); - for (const auto & child : children) - { - Coordination::Stat stat; - zookeeper.get(path + "/" + child, &stat); - if (!stat.ephemeralOwner) - { - ProfileEvents::increment(ProfileEvents::LeaderElectionAcquiredLeadership); - handler(); - return; - } - } + std::string node_path = node->getPath(); + node_name = node_path.substr(node_path.find_last_of('/') + 1); - zookeeper.create(path + "/leader_election-", identifier, CreateMode::PersistentSequential); task->activateAndSchedule(); } + void releaseNode() + { + shutdown(); + node = nullptr; + } + void threadFunction() { + bool success = false; + try { Strings children = zookeeper.getChildren(path); - if (children.empty()) - throw Poco::Exception("Assertion failed in LeaderElection"); - std::sort(children.begin(), children.end()); - Coordination::Stat stat; - zookeeper.get(path + "/" + children.front(), &stat); + auto my_node_it = std::lower_bound(children.begin(), children.end(), node_name); + if (my_node_it == children.end() || *my_node_it != node_name) + throw Poco::Exception("Assertion failed in LeaderElection"); - if (!stat.ephemeralOwner) + String value = zookeeper.get(path + "/" + children.front()); + +#if !defined(ARCADIA_BUILD) /// C++20; Replicated tables are unused in Arcadia. + if (value.ends_with(suffix)) { - /// It is persistent node - we can become leader. ProfileEvents::increment(ProfileEvents::LeaderElectionAcquiredLeadership); handler(); return; } +#endif + if (my_node_it == children.begin()) + throw Poco::Exception("Assertion failed in LeaderElection"); + + /// Watch for the node in front of us. + --my_node_it; + if (!zookeeper.existsWatch(path + "/" + *my_node_it, nullptr, task->getWatchCallback())) + task->schedule(); + + success = true; } catch (const KeeperException & e) { @@ -146,7 +155,8 @@ private: DB::tryLogCurrentException(log); } - task->scheduleAfter(10 * 1000); + if (!success) + task->scheduleAfter(10 * 1000); } }; From 689b6901f8ac076bcb11249a9a69303b2817e679 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 04:04:42 +0300 Subject: [PATCH 195/329] Fix typo --- src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index d861173d8a0..f867a39581f 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -298,7 +298,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( if (parts_to_merge.empty()) { if (out_disable_reason) - *out_disable_reason = "There are no need to merge parts according to merge selector algorithm"; + *out_disable_reason = "There is no need to merge parts according to merge selector algorithm"; return false; } From b51cbbdf15ea74218792d73168e8596b538f9802 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 04:08:56 +0300 Subject: [PATCH 196/329] Update test --- .../00620_optimize_on_nonleader_replica_zookeeper.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/00620_optimize_on_nonleader_replica_zookeeper.sql b/tests/queries/0_stateless/00620_optimize_on_nonleader_replica_zookeeper.sql index 9622a5bd3c2..f488502b13b 100644 --- a/tests/queries/0_stateless/00620_optimize_on_nonleader_replica_zookeeper.sql +++ b/tests/queries/0_stateless/00620_optimize_on_nonleader_replica_zookeeper.sql @@ -1,3 +1,5 @@ +-- The test is mostly outdated as now every replica is leader and can do OPTIMIZE locally. + DROP TABLE IF EXISTS rename1; DROP TABLE IF EXISTS rename2; DROP TABLE IF EXISTS rename3; @@ -14,7 +16,9 @@ SELECT * FROM rename1; RENAME TABLE rename2 TO rename3; INSERT INTO rename1 VALUES (0, 1, 2); +SYSTEM SYNC REPLICA rename3; -- Make "rename3" to see all data parts. OPTIMIZE TABLE rename3; +SYSTEM SYNC REPLICA rename1; -- Make "rename1" to see and process all scheduled merges. SELECT * FROM rename1; DROP TABLE IF EXISTS rename1; From 66ccb2f6b121b1e85cb035d6e6a256617722d4d3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 04:12:01 +0300 Subject: [PATCH 197/329] Remove "current_password" because it is harmful --- src/Interpreters/ClientInfo.h | 2 -- src/Interpreters/Context.cpp | 1 - src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.h | 2 ++ 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 7a4df63c17a..294eb47e3e9 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -47,8 +47,6 @@ public: String current_user; String current_query_id; Poco::Net::SocketAddress current_address; - /// Use current user and password when sending query to replica leader - String current_password; /// When query_kind == INITIAL_QUERY, these values are equal to current. String initial_user; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index cb780443e03..bd99039c36d 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -660,7 +660,6 @@ void Context::setUser(const String & name, const String & password, const Poco:: auto lock = getLock(); client_info.current_user = name; - client_info.current_password = password; client_info.current_address = address; auto new_user_id = getAccessControlManager().find(name); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ac762b3e05f..055e709cbc3 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4416,7 +4416,7 @@ void StorageReplicatedMergeTree::sendRequestToLeaderReplica(const ASTPtr & query const auto & query_settings = query_context.getSettingsRef(); const auto & query_client_info = query_context.getClientInfo(); String user = query_client_info.current_user; - String password = query_client_info.current_password; + String password; if (auto address = findClusterAddress(leader_address); address) { diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 5083abf7ef9..b2bd546b478 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -222,6 +222,7 @@ private: zkutil::EphemeralNodeHolderPtr replica_is_active_node; /** Is this replica "leading". The leader replica selects the parts to merge. + * It can be false only when old ClickHouse versions are working on the same cluster, because now we allow multiple leaders. */ std::atomic is_leader {false}; zkutil::LeaderElectionPtr leader_election; @@ -497,6 +498,7 @@ private: bool waitForReplicaToProcessLogEntry(const String & replica_name, const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active = true); /// Choose leader replica, send requst to it and wait. + /// Only makes sense when old ClickHouse versions are working on the same cluster, because now we allow multiple leaders. void sendRequestToLeaderReplica(const ASTPtr & query, const Context & query_context); /// Throw an exception if the table is readonly. From 5866401f6078010e51f31b6b2bed367c0bccca49 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 05:12:06 +0300 Subject: [PATCH 198/329] Less noise in cleanup thread --- .../ReplicatedMergeTreeCleanupThread.cpp | 75 +++++++++++++++---- .../ReplicatedMergeTreeCleanupThread.h | 3 +- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 1bc132eaba4..0870c0fdf72 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -85,8 +86,14 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() int children_count = stat.numChildren; - /// We will wait for 1.1 times more records to accumulate than necessary. - if (static_cast(children_count) < storage_settings->min_replicated_logs_to_keep * 1.1) + /// We will wait for 1.05 to 1.15 times more records to accumulate than necessary. + /// Randomization is needed to spread the time when multiple replicas come here. + /// Numbers are arbitrary. + std::uniform_real_distribution distr(1.05, 1.15); + double ratio = distr(rng); + size_t min_replicated_logs_to_keep = storage_settings->min_replicated_logs_to_keep * ratio; + + if (static_cast(children_count) < min_replicated_logs_to_keep) return; Strings replicas = zookeeper->getChildren(storage.zookeeper_path + "/replicas", &stat); @@ -214,10 +221,15 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() if (entries.empty()) return; - markLostReplicas(host_versions_lost_replicas, log_pointers_candidate_lost_replicas, replicas.size() - num_replicas_were_marked_is_lost, zookeeper); + markLostReplicas( + host_versions_lost_replicas, + log_pointers_candidate_lost_replicas, + replicas.size() - num_replicas_were_marked_is_lost, + zookeeper); Coordination::Requests ops; - for (size_t i = 0; i < entries.size(); ++i) + size_t i = 0; + for (; i < entries.size(); ++i) { ops.emplace_back(zkutil::makeRemoveRequest(storage.zookeeper_path + "/log/" + entries[i], -1)); @@ -229,12 +241,25 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() /// Simultaneously with clearing the log, we check to see if replica was added since we received replicas list. ops.emplace_back(zkutil::makeCheckRequest(storage.zookeeper_path + "/replicas", stat.version)); - zookeeper->multi(ops); + + try + { + zookeeper->multi(ops); + } + catch (const zkutil::KeeperMultiException & e) + { + /// Another replica already deleted the same node concurrently. + if (e.code == Coordination::Error::ZNONODE) + break; + + throw; + } ops.clear(); } } - LOG_DEBUG(log, "Removed {} old log entries: {} - {}", entries.size(), entries.front(), entries.back()); + if (i != 0) + LOG_DEBUG(log, "Removed {} old log entries: {} - {}", i, entries[0], entries[i - 1]); } @@ -250,8 +275,10 @@ void ReplicatedMergeTreeCleanupThread::markLostReplicas(const std::unordered_map String replica = pair.first; Coordination::Requests ops; /// If host changed version we can not mark replicas, because replica started to be active. - ops.emplace_back(zkutil::makeCheckRequest(storage.zookeeper_path + "/replicas/" + replica + "/host", host_versions_lost_replicas.at(replica))); - ops.emplace_back(zkutil::makeSetRequest(storage.zookeeper_path + "/replicas/" + replica + "/is_lost", "1", -1)); + ops.emplace_back(zkutil::makeCheckRequest( + storage.zookeeper_path + "/replicas/" + replica + "/host", host_versions_lost_replicas.at(replica))); + ops.emplace_back(zkutil::makeSetRequest( + storage.zookeeper_path + "/replicas/" + replica + "/is_lost", "1", -1)); candidate_lost_replicas.push_back(replica); requests.push_back(ops); } @@ -299,14 +326,17 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() /// Use ZooKeeper's first node (last according to time) timestamp as "current" time. Int64 current_time = timed_blocks.front().ctime; - Int64 time_threshold = std::max(static_cast(0), current_time - static_cast(1000 * storage_settings->replicated_deduplication_window_seconds)); + Int64 time_threshold = std::max( + static_cast(0), + current_time - static_cast(1000 * storage_settings->replicated_deduplication_window_seconds)); /// Virtual node, all nodes that are "greater" than this one will be deleted NodeWithStat block_threshold{{}, time_threshold}; size_t current_deduplication_window = std::min(timed_blocks.size(), storage_settings->replicated_deduplication_window); auto first_outdated_block_fixed_threshold = timed_blocks.begin() + current_deduplication_window; - auto first_outdated_block_time_threshold = std::upper_bound(timed_blocks.begin(), timed_blocks.end(), block_threshold, NodeWithStat::greaterByTime); + auto first_outdated_block_time_threshold = std::upper_bound( + timed_blocks.begin(), timed_blocks.end(), block_threshold, NodeWithStat::greaterByTime); auto first_outdated_block = std::min(first_outdated_block_fixed_threshold, first_outdated_block_time_threshold); zkutil::AsyncResponses try_remove_futures; @@ -326,13 +356,16 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() zookeeper->removeRecursive(path); cached_block_stats.erase(first_outdated_block->node); } - else if (rc != Coordination::Error::ZOK) - LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, Coordination::errorMessage(rc)); - else + else if (rc == Coordination::Error::ZOK || rc == Coordination::Error::ZNONODE) { + /// No node is Ok. Another replica is removing nodes concurrently. /// Successfully removed blocks have to be removed from cache cached_block_stats.erase(first_outdated_block->node); } + else + { + LOG_WARNING(log, "Error while deleting ZooKeeper path `{}`: {}, ignoring.", path, Coordination::errorMessage(rc)); + } first_outdated_block++; } @@ -453,8 +486,20 @@ void ReplicatedMergeTreeCleanupThread::clearOldMutations() { /// Simultaneously with clearing the log, we check to see if replica was added since we received replicas list. ops.emplace_back(zkutil::makeCheckRequest(storage.zookeeper_path + "/replicas", replicas_stat.version)); - zookeeper->multi(ops); - LOG_DEBUG(log, "Removed {} old mutation entries: {} - {}", (i + 1 - batch_start_i), entries[batch_start_i], entries[i]); + try + { + zookeeper->multi(ops); + } + catch (const zkutil::KeeperMultiException & e) + { + /// Another replica already deleted the same node concurrently. + if (e.code == Coordination::Error::ZNONODE) + break; + + throw; + } + LOG_DEBUG(log, "Removed {} old mutation entries: {} - {}", + i + 1 - batch_start_i, entries[batch_start_i], entries[i]); batch_start_i = i + 1; ops.clear(); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h index a787f99d907..f4191482d64 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -36,7 +37,7 @@ private: String log_name; Poco::Logger * log; BackgroundSchedulePool::TaskHolder task; - pcg64 rng; + pcg64 rng{randomSeed()}; void run(); void iterate(); From bbe5f4c9090d7dda7d79281f24b75e39a384aae0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 05:13:41 +0300 Subject: [PATCH 199/329] Revert "Remove "current_password" because it is harmful" This reverts commit 66ccb2f6b121b1e85cb035d6e6a256617722d4d3. --- src/Interpreters/ClientInfo.h | 2 ++ src/Interpreters/Context.cpp | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.h | 2 -- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 294eb47e3e9..7a4df63c17a 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -47,6 +47,8 @@ public: String current_user; String current_query_id; Poco::Net::SocketAddress current_address; + /// Use current user and password when sending query to replica leader + String current_password; /// When query_kind == INITIAL_QUERY, these values are equal to current. String initial_user; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index bd99039c36d..cb780443e03 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -660,6 +660,7 @@ void Context::setUser(const String & name, const String & password, const Poco:: auto lock = getLock(); client_info.current_user = name; + client_info.current_password = password; client_info.current_address = address; auto new_user_id = getAccessControlManager().find(name); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 055e709cbc3..ac762b3e05f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4416,7 +4416,7 @@ void StorageReplicatedMergeTree::sendRequestToLeaderReplica(const ASTPtr & query const auto & query_settings = query_context.getSettingsRef(); const auto & query_client_info = query_context.getClientInfo(); String user = query_client_info.current_user; - String password; + String password = query_client_info.current_password; if (auto address = findClusterAddress(leader_address); address) { diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index b2bd546b478..5083abf7ef9 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -222,7 +222,6 @@ private: zkutil::EphemeralNodeHolderPtr replica_is_active_node; /** Is this replica "leading". The leader replica selects the parts to merge. - * It can be false only when old ClickHouse versions are working on the same cluster, because now we allow multiple leaders. */ std::atomic is_leader {false}; zkutil::LeaderElectionPtr leader_election; @@ -498,7 +497,6 @@ private: bool waitForReplicaToProcessLogEntry(const String & replica_name, const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active = true); /// Choose leader replica, send requst to it and wait. - /// Only makes sense when old ClickHouse versions are working on the same cluster, because now we allow multiple leaders. void sendRequestToLeaderReplica(const ASTPtr & query, const Context & query_context); /// Throw an exception if the table is readonly. From d2c66f96881bcdc18248711a2db3ed6e953437c3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 05:14:59 +0300 Subject: [PATCH 200/329] Added comments --- src/Storages/StorageReplicatedMergeTree.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 5083abf7ef9..b2bd546b478 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -222,6 +222,7 @@ private: zkutil::EphemeralNodeHolderPtr replica_is_active_node; /** Is this replica "leading". The leader replica selects the parts to merge. + * It can be false only when old ClickHouse versions are working on the same cluster, because now we allow multiple leaders. */ std::atomic is_leader {false}; zkutil::LeaderElectionPtr leader_election; @@ -497,6 +498,7 @@ private: bool waitForReplicaToProcessLogEntry(const String & replica_name, const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active = true); /// Choose leader replica, send requst to it and wait. + /// Only makes sense when old ClickHouse versions are working on the same cluster, because now we allow multiple leaders. void sendRequestToLeaderReplica(const ASTPtr & query, const Context & query_context); /// Throw an exception if the table is readonly. From 1ab599b0a0598be53720765bebc0565d222addaa Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 05:17:08 +0300 Subject: [PATCH 201/329] Remove "current_password" but keep it for Arcadians --- src/Interpreters/ClientInfo.h | 3 ++- src/Interpreters/Context.cpp | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 7a4df63c17a..704fba3b3ef 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -47,7 +47,8 @@ public: String current_user; String current_query_id; Poco::Net::SocketAddress current_address; - /// Use current user and password when sending query to replica leader + + /// This field is only used in foreign "Arcadia" build. String current_password; /// When query_kind == INITIAL_QUERY, these values are equal to current. diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index cb780443e03..02060534aef 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -660,9 +660,13 @@ void Context::setUser(const String & name, const String & password, const Poco:: auto lock = getLock(); client_info.current_user = name; - client_info.current_password = password; client_info.current_address = address; +#if defined(ARCADIA_BUILD) + /// This is harmful field that is used only in foreign "Arcadia" build. + client_info.current_password = password; +#endif + auto new_user_id = getAccessControlManager().find(name); std::shared_ptr new_access; if (new_user_id) From 0c1b2d48a30d4006271638de056070ac9cc5a4ee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 06:58:07 +0300 Subject: [PATCH 202/329] Update test --- .../01249_bad_arguments_for_bloom_filter.reference | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference index 70d176d9b7a..e3f4955d4cf 100644 --- a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference +++ b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference @@ -1,3 +1,3 @@ -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64, \n `i32` Int32, \n `f64` Float64, \n `d` Decimal(10, 2), \n `s` String, \n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), \n `dt` Date, \n INDEX bloom_filter_a i32 TYPE bloom_filter(0., 1.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64, \n `i32` Int32, \n `f64` Float64, \n `d` Decimal(10, 2), \n `s` String, \n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), \n `dt` Date, \n INDEX bloom_filter_a i32 TYPE bloom_filter(-0.1) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64, \n `i32` Int32, \n `f64` Float64, \n `d` Decimal(10, 2), \n `s` String, \n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3), \n `dt` Date, \n INDEX bloom_filter_a i32 TYPE bloom_filter(1.01) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(0., 1.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(-0.1) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(1.01) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 From 1c5c2f8c690a714e4f53b7809813364989789ef9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 07:08:20 +0300 Subject: [PATCH 203/329] Fix formatting of CREATE DICTIONARY --- src/Parsers/ASTDictionaryAttributeDeclaration.cpp | 3 --- src/Parsers/ParserCreateQuery.cpp | 1 - 2 files changed, 4 deletions(-) diff --git a/src/Parsers/ASTDictionaryAttributeDeclaration.cpp b/src/Parsers/ASTDictionaryAttributeDeclaration.cpp index 2b056cb3743..05ba48ace7b 100644 --- a/src/Parsers/ASTDictionaryAttributeDeclaration.cpp +++ b/src/Parsers/ASTDictionaryAttributeDeclaration.cpp @@ -34,9 +34,6 @@ void ASTDictionaryAttributeDeclaration::formatImpl(const FormatSettings & settin { frame.need_parens = false; - if (!settings.one_line) - settings.ostr << settings.nl_or_ws << std::string(4 * frame.indent, ' '); - settings.ostr << backQuote(name); if (type) diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index c54033bd27d..f8c137fb679 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -796,7 +796,6 @@ bool ParserCreateDictionaryQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, E ParserDictionaryAttributeDeclarationList attributes_p; ParserDictionary dictionary_p; - bool if_not_exists = false; ASTPtr database; From e9eb722d4ac539539fd8a1b803b3b203fa42d91b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 07:27:33 +0300 Subject: [PATCH 204/329] Better formatting of CREATE queries --- src/Parsers/ASTCreateQuery.cpp | 1 + src/Parsers/ASTExpressionList.cpp | 6 ++++-- src/Parsers/IAST.h | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index f7481ac3c09..fb6bbaeafb0 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -197,6 +197,7 @@ ASTPtr ASTCreateQuery::clone() const void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; + frame.expression_list_always_start_on_new_line = true; if (!database.empty() && table.empty()) { diff --git a/src/Parsers/ASTExpressionList.cpp b/src/Parsers/ASTExpressionList.cpp index 1395d8b15fe..abab1e895cf 100644 --- a/src/Parsers/ASTExpressionList.cpp +++ b/src/Parsers/ASTExpressionList.cpp @@ -39,10 +39,12 @@ void ASTExpressionList::formatImplMultiline(const FormatSettings & settings, For settings.ostr << separator; } - if (children.size() > 1) + if (children.size() > 1 || frame.expression_list_always_start_on_new_line) settings.ostr << indent_str; - (*it)->formatImpl(settings, state, frame); + FormatStateStacked frame_nested = frame; + frame_nested.expression_list_always_start_on_new_line = false; + (*it)->formatImpl(settings, state, frame_nested); } } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 88dedc54d3f..c0c286ac0d2 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -202,6 +202,7 @@ public: { UInt8 indent = 0; bool need_parens = false; + bool expression_list_always_start_on_new_line = false; /// Line feed and indent before expression list even if it's of single element. const IAST * current_select = nullptr; }; From e2607f005c334ee8e35b107308b016bd98db7412 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 07:36:20 +0300 Subject: [PATCH 205/329] Fix error with ALTER CONSTRAINT formatting; added a test --- src/Parsers/ASTAlterQuery.cpp | 2 +- .../01318_alter_add_constraint_format.reference | 1 + .../0_stateless/01318_alter_add_constraint_format.sh | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01318_alter_add_constraint_format.reference create mode 100755 tests/queries/0_stateless/01318_alter_add_constraint_format.sh diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index f323f66be17..1309037ec01 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -146,7 +146,7 @@ void ASTAlterCommand::formatImpl( } else if (type == ASTAlterCommand::ADD_CONSTRAINT) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "ADD CONSTRAINT" << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "ADD CONSTRAINT " << (if_not_exists ? "IF NOT EXISTS " : "") << (settings.hilite ? hilite_none : ""); constraint_decl->formatImpl(settings, state, frame); } else if (type == ASTAlterCommand::DROP_CONSTRAINT) diff --git a/tests/queries/0_stateless/01318_alter_add_constraint_format.reference b/tests/queries/0_stateless/01318_alter_add_constraint_format.reference new file mode 100644 index 00000000000..4283da7b3af --- /dev/null +++ b/tests/queries/0_stateless/01318_alter_add_constraint_format.reference @@ -0,0 +1 @@ +ALTER TABLE replicated_constraints1 ADD CONSTRAINT IF NOT EXISTS b_constraint CHECK b > 10 diff --git a/tests/queries/0_stateless/01318_alter_add_constraint_format.sh b/tests/queries/0_stateless/01318_alter_add_constraint_format.sh new file mode 100755 index 00000000000..f8eb655a766 --- /dev/null +++ b/tests/queries/0_stateless/01318_alter_add_constraint_format.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_FORMAT --oneline <<<"ALTER TABLE replicated_constraints1 ADD CONSTRAINT IF NOT EXISTS b_constraint CHECK b > 10" From d5e3e7ff761d9fa56b5b6ad75dd81516e45a043f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 07:36:55 +0300 Subject: [PATCH 206/329] Update tests --- .../queries/0_stateless/01018_ddl_dictionaries_create.reference | 2 +- .../01110_dictionary_layout_without_arguments.reference | 2 +- .../0_stateless/01224_no_superfluous_dict_reload.reference | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference b/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference index ad16e8ae7f2..7c2eca9cedf 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference @@ -1,5 +1,5 @@ =DICTIONARY in Ordinary DB -CREATE DICTIONARY ordinary_db.dict1\n(\n `key_column` UInt64 DEFAULT 0, \n `second_column` UInt8 DEFAULT 1, \n `third_column` String DEFAULT \'qqq\'\n)\nPRIMARY KEY key_column\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'database_for_dict\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT()) +CREATE DICTIONARY ordinary_db.dict1\n(\n `key_column` UInt64 DEFAULT 0,\n `second_column` UInt8 DEFAULT 1,\n `third_column` String DEFAULT \'qqq\'\n)\nPRIMARY KEY key_column\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'database_for_dict\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT()) dict1 1 ordinary_db dict1 diff --git a/tests/queries/0_stateless/01110_dictionary_layout_without_arguments.reference b/tests/queries/0_stateless/01110_dictionary_layout_without_arguments.reference index 852abeea187..69018bef2ef 100644 --- a/tests/queries/0_stateless/01110_dictionary_layout_without_arguments.reference +++ b/tests/queries/0_stateless/01110_dictionary_layout_without_arguments.reference @@ -1,3 +1,3 @@ World -CREATE DICTIONARY db_for_dict.dict_with_hashed_layout\n(\n `key1` UInt64, \n `value` String\n)\nPRIMARY KEY key1\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' DB \'db_for_dict\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(HASHED) +CREATE DICTIONARY db_for_dict.dict_with_hashed_layout\n(\n `key1` UInt64,\n `value` String\n)\nPRIMARY KEY key1\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' DB \'db_for_dict\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(HASHED) Hello diff --git a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference index 96d4393e06b..d80501b3f4d 100644 --- a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference +++ b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.reference @@ -2,7 +2,7 @@ NOT_LOADED NOT_LOADED CREATE DICTIONARY dict_db_01224.dict ( - `key` UInt64 DEFAULT 0, + `key` UInt64 DEFAULT 0, `val` UInt64 DEFAULT 10 ) PRIMARY KEY key From 94d55abfd13d0a3c0e4299f96e80fabb75b35a01 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 07:40:03 +0300 Subject: [PATCH 207/329] Update tests --- src/Parsers/ASTCreateQuery.cpp | 5 ++++- .../0_stateless/00933_ttl_replicated_zookeeper.reference | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index fb6bbaeafb0..201e2e45528 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -197,7 +197,6 @@ ASTPtr ASTCreateQuery::clone() const void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { frame.need_parens = false; - frame.expression_list_always_start_on_new_line = true; if (!database.empty() && table.empty()) { @@ -271,6 +270,8 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat << (!as_database.empty() ? backQuoteIfNeed(as_database) + "." : "") << backQuoteIfNeed(as_table); } + frame.expression_list_always_start_on_new_line = true; + if (columns_list) { settings.ostr << (settings.one_line ? " (" : "\n("); @@ -290,6 +291,8 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat settings.ostr << (settings.one_line ? ")" : "\n)"); } + frame.expression_list_always_start_on_new_line = false; + if (storage) storage->formatImpl(settings, state, frame); diff --git a/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference b/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference index 629fbf2a4a3..c727c24707d 100644 --- a/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/00933_ttl_replicated_zookeeper.reference @@ -1,3 +1,3 @@ 200 400 -CREATE TABLE test.ttl_repl2\n(\n `d` Date, \n `x` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/ttl_repl\', \'2\')\nPARTITION BY toDayOfMonth(d)\nORDER BY x\nTTL d + toIntervalDay(1)\nSETTINGS index_granularity = 8192 +CREATE TABLE test.ttl_repl2\n(\n `d` Date,\n `x` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/ttl_repl\', \'2\')\nPARTITION BY toDayOfMonth(d)\nORDER BY x\nTTL d + toIntervalDay(1)\nSETTINGS index_granularity = 8192 From e99c6d9143163bcc5104d391d620e18b4aaf83a0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 07:42:29 +0300 Subject: [PATCH 208/329] Update tests --- ...cated_merge_tree_alter_zookeeper.reference | 48 +++++++++---------- ...om_compression_codecs_replicated.reference | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference b/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference index fa5e65d2d60..ac0e0d557cb 100644 --- a/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference +++ b/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference @@ -1,22 +1,22 @@ d Date k UInt64 i32 Int32 -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 10 42 d Date k UInt64 i32 Int32 dt DateTime -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 9 41 1992-01-01 08:00:00 2015-01-01 10 42 0000-00-00 00:00:00 d Date @@ -25,14 +25,14 @@ i32 Int32 dt DateTime n.ui8 Array(UInt8) n.s Array(String) -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.ui8` Array(UInt8),\n `n.s` Array(String)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime n.ui8 Array(UInt8) n.s Array(String) -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.ui8` Array(UInt8),\n `n.s` Array(String)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] 2015-01-01 10 42 0000-00-00 00:00:00 [] [] @@ -43,7 +43,7 @@ dt DateTime n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -51,7 +51,7 @@ dt DateTime n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] [] @@ -64,7 +64,7 @@ n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) s String DEFAULT \'0\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date), \n `s` String DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date),\n `s` String DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -73,7 +73,7 @@ n.ui8 Array(UInt8) n.s Array(String) n.d Array(Date) s String DEFAULT \'0\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `n.d` Array(Date), \n `s` String DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `n.d` Array(Date),\n `s` String DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] ['2000-01-01','2000-01-01','2000-01-03'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 0 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 0 @@ -86,7 +86,7 @@ dt DateTime n.ui8 Array(UInt8) n.s Array(String) s Int64 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` Int64 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` Int64 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -94,7 +94,7 @@ dt DateTime n.ui8 Array(UInt8) n.s Array(String) s Int64 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` Int64 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` Int64 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] 0 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 0 @@ -108,7 +108,7 @@ n.ui8 Array(UInt8) n.s Array(String) s UInt32 DEFAULT \'0\' n.d Array(Date) -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\', \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\',\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -117,7 +117,7 @@ n.ui8 Array(UInt8) n.s Array(String) s UInt32 DEFAULT \'0\' n.d Array(Date) -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.ui8` Array(UInt8), \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\', \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.ui8` Array(UInt8),\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\',\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] 0 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 0 ['0000-00-00','0000-00-00','0000-00-00'] @@ -129,14 +129,14 @@ i32 Int32 dt DateTime n.s Array(String) s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime n.s Array(String) s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `n.s` Array(String), \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `n.s` Array(String),\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 ['asd','qwe','qwe'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 ['120','130','140'] 0 2015-01-01 8 40 2012-12-12 12:12:12 ['12','13','14'] 0 @@ -147,13 +147,13 @@ k UInt64 i32 Int32 dt DateTime s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 2015-01-01 7 39 2014-07-14 13:26:50 0 2015-01-01 8 40 2012-12-12 12:12:12 0 @@ -166,7 +166,7 @@ dt DateTime s UInt32 DEFAULT \'0\' n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\', \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `s` UInt32 DEFAULT \'0\',\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 @@ -174,7 +174,7 @@ dt DateTime s UInt32 DEFAULT \'0\' n.s Array(String) n.d Array(Date) -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\', \n `n.s` Array(String), \n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `s` UInt32 DEFAULT \'0\',\n `n.s` Array(String),\n `n.d` Array(Date)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 [] [] 2015-01-01 7 39 2014-07-14 13:26:50 0 [] [] 2015-01-01 8 40 2012-12-12 12:12:12 0 [] [] @@ -185,13 +185,13 @@ k UInt64 i32 Int32 dt DateTime s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt DateTime s UInt32 DEFAULT \'0\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` DateTime, \n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` DateTime,\n `s` UInt32 DEFAULT \'0\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 2015-01-01 7 39 2014-07-14 13:26:50 0 2015-01-01 8 40 2012-12-12 12:12:12 0 @@ -202,13 +202,13 @@ k UInt64 i32 Int32 dt Date s DateTime DEFAULT \'0000-00-00 00:00:00\' -CREATE TABLE test.replicated_alter1\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` Date, \n `s` DateTime DEFAULT \'0000-00-00 00:00:00\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) +CREATE TABLE test.replicated_alter1\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` Date,\n `s` DateTime DEFAULT \'0000-00-00 00:00:00\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) d Date k UInt64 i32 Int32 dt Date s DateTime DEFAULT \'0000-00-00 00:00:00\' -CREATE TABLE test.replicated_alter2\n(\n `d` Date, \n `k` UInt64, \n `i32` Int32, \n `dt` Date, \n `s` DateTime DEFAULT \'0000-00-00 00:00:00\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) +CREATE TABLE test.replicated_alter2\n(\n `d` Date,\n `k` UInt64,\n `i32` Int32,\n `dt` Date,\n `s` DateTime DEFAULT \'0000-00-00 00:00:00\'\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 1970-01-02 06:55:00 2015-01-01 7 39 2014-07-14 0000-00-00 00:00:00 2015-01-01 8 40 2012-12-12 0000-00-00 00:00:00 diff --git a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference index ee481c88d89..62cea01089a 100644 --- a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference +++ b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated.reference @@ -20,7 +20,7 @@ 274972506.6 9175437371954010821 9175437371954010821 -CREATE TABLE test.compression_codec_multiple_more_types_replicated\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)), \n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)), \n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)), \n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/compression_codec_multiple_more_types_replicated\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE test.compression_codec_multiple_more_types_replicated\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)),\n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)),\n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)),\n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/compression_codec_multiple_more_types_replicated\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1.5555555555555 hello world! [77] ['John'] 7.1000000000000 xxxxxxxxxxxx [127] ['Henry'] ! From 6ddc6d7f085aae86e5ed261be4788a6a19db66cf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 07:51:27 +0300 Subject: [PATCH 209/329] Make the test faster #11637 --- tests/queries/0_stateless/01307_multiple_leaders.reference | 4 ++-- tests/queries/0_stateless/01307_multiple_leaders.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01307_multiple_leaders.reference b/tests/queries/0_stateless/01307_multiple_leaders.reference index 576441b288d..62cda31dff8 100644 --- a/tests/queries/0_stateless/01307_multiple_leaders.reference +++ b/tests/queries/0_stateless/01307_multiple_leaders.reference @@ -1,2 +1,2 @@ -2000 1999000 -2000 1999000 +400 79800 +400 79800 diff --git a/tests/queries/0_stateless/01307_multiple_leaders.sh b/tests/queries/0_stateless/01307_multiple_leaders.sh index 0bf5e0b13bf..e19a10bcecb 100755 --- a/tests/queries/0_stateless/01307_multiple_leaders.sh +++ b/tests/queries/0_stateless/01307_multiple_leaders.sh @@ -22,8 +22,8 @@ function thread() } -thread 0 1000 & -thread 1 1000 & +thread 0 200 & +thread 1 200 & wait From 65a8fe7cf053b9209c591c249971c8a8b9e4a102 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 09:14:58 +0300 Subject: [PATCH 210/329] Update tests --- .../00725_ipv4_ipv6_domains.reference | 4 +-- .../01069_database_memory.reference | 2 +- ..._expressions_in_engine_arguments.reference | 12 +++---- .../01272_suspicious_codecs.reference | 32 +++++++++---------- .../01297_alter_distributed.reference | 4 +-- .../0_stateless/01298_alter_merge.reference | 4 +-- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference b/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference index 69804e6cd24..28051d15f65 100644 --- a/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference +++ b/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference @@ -1,4 +1,4 @@ -CREATE TABLE default.ipv4_test\n(`ipv4_` IPv4\n)\nENGINE = Memory +CREATE TABLE default.ipv4_test\n(\n `ipv4_` IPv4\n)\nENGINE = Memory 0.0.0.0 00 8.8.8.8 08080808 127.0.0.1 7F000001 @@ -10,7 +10,7 @@ CREATE TABLE default.ipv4_test\n(`ipv4_` IPv4\n)\nENGINE = Memory > 127.0.0.1 255.255.255.255 = 127.0.0.1 127.0.0.1 euqality of IPv4-mapped IPv6 value and IPv4 promoted to IPv6 with function: 1 -CREATE TABLE default.ipv6_test\n(`ipv6_` IPv6\n)\nENGINE = Memory +CREATE TABLE default.ipv6_test\n(\n `ipv6_` IPv6\n)\nENGINE = Memory :: 00000000000000000000000000000000 :: 00000000000000000000000000000000 ::ffff:8.8.8.8 00000000000000000000FFFF08080808 diff --git a/tests/queries/0_stateless/01069_database_memory.reference b/tests/queries/0_stateless/01069_database_memory.reference index cfccf5b1757..e7486d57276 100644 --- a/tests/queries/0_stateless/01069_database_memory.reference +++ b/tests/queries/0_stateless/01069_database_memory.reference @@ -5,4 +5,4 @@ CREATE DATABASE memory_01069\nENGINE = Memory() 4 3 4 -CREATE TABLE memory_01069.file\n(`n` UInt8\n)\nENGINE = File(\'CSV\') +CREATE TABLE memory_01069.file\n(\n `n` UInt8\n)\nENGINE = File(\'CSV\') diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference index 138f09f2634..d360a046958 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference @@ -1,11 +1,11 @@ -CREATE TABLE test_01083.file\n(`n` Int8\n)\nENGINE = File(\'TSVWithNamesAndTypes\') -CREATE TABLE test_01083.buffer\n(`n` Int8\n)\nENGINE = Buffer(\'test_01083\', \'file\', 16, 10, 200, 10000, 1000000, 10000000, 1000000000) -CREATE TABLE test_01083.merge\n(`n` Int8\n)\nENGINE = Merge(\'test_01083\', \'distributed\') +CREATE TABLE test_01083.file\n(\n `n` Int8\n)\nENGINE = File(\'TSVWithNamesAndTypes\') +CREATE TABLE test_01083.buffer\n(\n `n` Int8\n)\nENGINE = Buffer(\'test_01083\', \'file\', 16, 10, 200, 10000, 1000000, 10000000, 1000000000) +CREATE TABLE test_01083.merge\n(\n `n` Int8\n)\nENGINE = Merge(\'test_01083\', \'distributed\') CREATE TABLE test_01083.merge_tf AS merge(\'test_01083\', \'.*\') -CREATE TABLE test_01083.distributed\n(`n` Int8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'test_01083\', \'file\') +CREATE TABLE test_01083.distributed\n(\n `n` Int8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'test_01083\', \'file\') CREATE TABLE test_01083.distributed_tf AS cluster(\'test_shard_localhost\', \'test_01083\', \'buffer\') CREATE TABLE test_01083.url\n(\n `n` UInt64,\n `col` String\n)\nENGINE = URL(\'https://localhost:8443/?query=select+n,+_table+from+test_01083.merge+format+CSV\', \'CSV\') CREATE TABLE test_01083.rich_syntax AS remote(\'localhos{x|y|t}\', cluster(\'test_shard_localhost\', remote(\'127.0.0.{1..4}\', \'test_01083\', \'view\'))) -CREATE VIEW test_01083.view\n(`n` Int64\n) AS\nSELECT toInt64(n) AS n\nFROM \n(\n SELECT toString(n) AS n\n FROM test_01083.merge\n WHERE _table != \'qwerty\'\n ORDER BY _table ASC\n)\nUNION ALL\nSELECT *\nFROM test_01083.file -CREATE DICTIONARY test_01083.dict\n(\n \n `n` UInt64,\n \n `col` String DEFAULT \'42\'\n)\nPRIMARY KEY n\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9440 SECURE 1 USER \'default\' TABLE \'url\' DB \'test_01083\'))\nLIFETIME(MIN 0 MAX 1)\nLAYOUT(CACHE(SIZE_IN_CELLS 1)) +CREATE VIEW test_01083.view\n(\n `n` Int64\n) AS\nSELECT toInt64(n) AS n\nFROM \n(\n SELECT toString(n) AS n\n FROM test_01083.merge\n WHERE _table != \'qwerty\'\n ORDER BY _table ASC\n)\nUNION ALL\nSELECT *\nFROM test_01083.file +CREATE DICTIONARY test_01083.dict\n(\n `n` UInt64,\n `col` String DEFAULT \'42\'\n)\nPRIMARY KEY n\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9440 SECURE 1 USER \'default\' TABLE \'url\' DB \'test_01083\'))\nLIFETIME(MIN 0 MAX 1)\nLAYOUT(CACHE(SIZE_IN_CELLS 1)) 16 diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.reference b/tests/queries/0_stateless/01272_suspicious_codecs.reference index de91a1ddb25..559b6df2693 100644 --- a/tests/queries/0_stateless/01272_suspicious_codecs.reference +++ b/tests/queries/0_stateless/01272_suspicious_codecs.reference @@ -1,16 +1,16 @@ -CREATE TABLE default.codecs1\n(`a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs2\n(`a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs3\n(`a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs4\n(`a` UInt8 CODEC(LZ4, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs5\n(`a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs6\n(`a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs7\n(`a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs8\n(`a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs1\n(`a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs2\n(`a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs3\n(`a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs4\n(`a` UInt8 CODEC(LZ4, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs5\n(`a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs6\n(`a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs7\n(`a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -CREATE TABLE default.codecs8\n(`a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs1\n(\n `a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs2\n(\n `a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs3\n(\n `a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs4\n(\n `a` UInt8 CODEC(LZ4, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs5\n(\n `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs6\n(\n `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs7\n(\n `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs8\n(\n `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs1\n(\n `a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs2\n(\n `a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs3\n(\n `a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs4\n(\n `a` UInt8 CODEC(LZ4, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs5\n(\n `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs6\n(\n `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs7\n(\n `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs8\n(\n `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01297_alter_distributed.reference b/tests/queries/0_stateless/01297_alter_distributed.reference index bd269322884..8fd8bc7ab72 100644 --- a/tests/queries/0_stateless/01297_alter_distributed.reference +++ b/tests/queries/0_stateless/01297_alter_distributed.reference @@ -6,7 +6,7 @@ VisitID UInt64 UserID UInt64 StartTime DateTime ClickLogID UInt64 -CREATE TABLE default.merge_distributed\n(\n `CounterID` UInt32, \n `dummy` String, \n `StartDate` Date, \n `Sign` Int8, \n `VisitID` UInt64, \n `UserID` UInt64, \n `StartTime` DateTime, \n `ClickLogID` UInt64\n)\nENGINE = Distributed(\'test_shard_localhost\', \'default\', \'merge_distributed1\') +CREATE TABLE default.merge_distributed\n(\n `CounterID` UInt32,\n `dummy` String,\n `StartDate` Date,\n `Sign` Int8,\n `VisitID` UInt64,\n `UserID` UInt64,\n `StartTime` DateTime,\n `ClickLogID` UInt64\n)\nENGINE = Distributed(\'test_shard_localhost\', \'default\', \'merge_distributed1\') 1 Hello, Alter Table! CounterID UInt32 StartDate Date @@ -15,4 +15,4 @@ VisitID UInt64 UserID UInt64 StartTime DateTime ClickLogID UInt64 -CREATE TABLE default.merge_distributed\n(\n `CounterID` UInt32, \n `StartDate` Date, \n `Sign` Int8, \n `VisitID` UInt64, \n `UserID` UInt64, \n `StartTime` DateTime, \n `ClickLogID` UInt64\n)\nENGINE = Distributed(\'test_shard_localhost\', \'default\', \'merge_distributed1\') +CREATE TABLE default.merge_distributed\n(\n `CounterID` UInt32,\n `StartDate` Date,\n `Sign` Int8,\n `VisitID` UInt64,\n `UserID` UInt64,\n `StartTime` DateTime,\n `ClickLogID` UInt64\n)\nENGINE = Distributed(\'test_shard_localhost\', \'default\', \'merge_distributed1\') diff --git a/tests/queries/0_stateless/01298_alter_merge.reference b/tests/queries/0_stateless/01298_alter_merge.reference index 393c0a600ff..a012900f978 100644 --- a/tests/queries/0_stateless/01298_alter_merge.reference +++ b/tests/queries/0_stateless/01298_alter_merge.reference @@ -6,7 +6,7 @@ VisitID UInt64 UserID UInt64 StartTime DateTime ClickLogID UInt64 -CREATE TABLE default.merge\n(\n `CounterID` UInt32, \n `dummy` String, \n `StartDate` Date, \n `Sign` Int8, \n `VisitID` UInt64, \n `UserID` UInt64, \n `StartTime` DateTime, \n `ClickLogID` UInt64\n)\nENGINE = Merge(\'default\', \'merge\\\\[0-9\\\\]\') +CREATE TABLE default.merge\n(\n `CounterID` UInt32,\n `dummy` String,\n `StartDate` Date,\n `Sign` Int8,\n `VisitID` UInt64,\n `UserID` UInt64,\n `StartTime` DateTime,\n `ClickLogID` UInt64\n)\nENGINE = Merge(\'default\', \'merge\\\\[0-9\\\\]\') CounterID UInt32 StartDate Date Sign Int8 @@ -14,4 +14,4 @@ VisitID UInt64 UserID UInt64 StartTime DateTime ClickLogID UInt64 -CREATE TABLE default.merge\n(\n `CounterID` UInt32, \n `StartDate` Date, \n `Sign` Int8, \n `VisitID` UInt64, \n `UserID` UInt64, \n `StartTime` DateTime, \n `ClickLogID` UInt64\n)\nENGINE = Merge(\'default\', \'merge\\\\[0-9\\\\]\') +CREATE TABLE default.merge\n(\n `CounterID` UInt32,\n `StartDate` Date,\n `Sign` Int8,\n `VisitID` UInt64,\n `UserID` UInt64,\n `StartTime` DateTime,\n `ClickLogID` UInt64\n)\nENGINE = Merge(\'default\', \'merge\\\\[0-9\\\\]\') From eccd8d61dd5b67220267a171841827bfebcc2eca Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 15 Jun 2020 10:13:29 +0300 Subject: [PATCH 211/329] Update build.py --- docs/tools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 95e887f046f..b7ddbc29629 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -220,7 +220,7 @@ if __name__ == '__main__': arg_parser.add_argument('--website-dir', default=website_dir) arg_parser.add_argument('--output-dir', default='build') arg_parser.add_argument('--enable-stable-releases', action='store_true') - arg_parser.add_argument('--stable-releases-limit', type=int, default='4') + arg_parser.add_argument('--stable-releases-limit', type=int, default='3') arg_parser.add_argument('--lts-releases-limit', type=int, default='2') arg_parser.add_argument('--nav-limit', type=int, default='0') arg_parser.add_argument('--version-prefix', type=str, default='') From 013c03a70918067bc08f3e1f2dbac0a63533590a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2020 07:54:48 +0000 Subject: [PATCH 212/329] Bump googletrans from 2.4.0 to 3.0.0 in /docs/tools Bumps [googletrans](https://github.com/ssut/py-googletrans) from 2.4.0 to 3.0.0. - [Release notes](https://github.com/ssut/py-googletrans/releases) - [Commits](https://github.com/ssut/py-googletrans/commits) Signed-off-by: dependabot-preview[bot] --- docs/tools/translate/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/translate/requirements.txt b/docs/tools/translate/requirements.txt index 0c9d44a346e..370554fa90f 100644 --- a/docs/tools/translate/requirements.txt +++ b/docs/tools/translate/requirements.txt @@ -1,7 +1,7 @@ Babel==2.8.0 certifi==2020.4.5.2 chardet==3.0.4 -googletrans==2.4.0 +googletrans==3.0.0 idna==2.9 Jinja2==2.11.2 pandocfilters==1.4.2 From c20ce687cf21bfc2015a7bf8f0c3bb92e81b52ce Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Jun 2020 11:12:01 +0300 Subject: [PATCH 213/329] bump ci --- src/Storages/AlterCommands.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 8e8b308ac3d..32b6a94ff23 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -716,7 +716,7 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, const Context & co if (!command.ignore) command.apply(metadata_copy, context); - /// Changes in columns may lead to changes in keys expression + /// Changes in columns may lead to changes in keys expression. metadata_copy.sorting_key.recalculateWithNewColumns(metadata_copy.columns, context); if (metadata_copy.primary_key.definition_ast != nullptr) { @@ -728,7 +728,7 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, const Context & co metadata_copy.primary_key.definition_ast = nullptr; } - /// Changes in columns may lead to changes in TTL expressions + /// Changes in columns may lead to changes in TTL expressions. auto column_ttl_asts = metadata_copy.columns.getColumnTTLs(); for (const auto & [name, ast] : column_ttl_asts) { From 2c9ce0f3fa4e7f02294bbb6e6021eee3633cc289 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Jun 2020 13:14:36 +0300 Subject: [PATCH 214/329] Bump CI --- src/Storages/AlterCommands.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 32b6a94ff23..c0fc53aa8e3 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -327,7 +327,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con primary_key = KeyDescription::getKeyFromAST(sorting_key.definition_ast, metadata.columns, context); } - /// Recalculate key with new order_by expression + /// Recalculate key with new order_by expression. sorting_key.recalculateWithNewAST(order_by, metadata.columns, context); } else if (type == COMMENT_COLUMN) From def0158638b82c6d2d38ceb80daec6f74b992a15 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 15 Jun 2020 14:33:44 +0300 Subject: [PATCH 215/329] configure query handler as default --- src/Server/HTTPHandlerFactory.cpp | 31 ++++++++++++------- src/Server/HTTPHandlerFactory.h | 2 -- .../test_http_handlers_config/test.py | 3 ++ 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp index 6459b0aab3b..f34852054d1 100644 --- a/src/Server/HTTPHandlerFactory.cpp +++ b/src/Server/HTTPHandlerFactory.cpp @@ -20,6 +20,9 @@ namespace ErrorCodes extern const int INVALID_CONFIG_PARAMETER; } +static void addCommonDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server); +static void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics & async_metrics); + HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(const std::string & name_) : log(&Poco::Logger::get(name_)), name(name_) { @@ -75,7 +78,7 @@ static inline auto createHandlersFactoryFromConfig( for (const auto & key : keys) { if (key == "defaults") - addDefaultHandlersFactory(*main_handler_factory, server, &async_metrics); + addDefaultHandlersFactory(*main_handler_factory, server, async_metrics); else if (startsWith(key, "rule")) { const auto & handler_type = server.config().getString(prefix + "." + key + ".handler.type", ""); @@ -113,12 +116,7 @@ static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory(IS else { auto factory = std::make_unique(name); - addDefaultHandlersFactory(*factory, server, &async_metrics); - - auto query_handler = std::make_unique>(server, "query"); - query_handler->allowPostAndGetParamsRequest(); - factory->addHandler(query_handler.release()); - + addDefaultHandlersFactory(*factory, server, async_metrics); return factory.release(); } } @@ -126,7 +124,7 @@ static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory(IS static inline Poco::Net::HTTPRequestHandlerFactory * createInterserverHTTPHandlerFactory(IServer & server, const std::string & name) { auto factory = std::make_unique(name); - addDefaultHandlersFactory(*factory, server, nullptr); + addCommonDefaultHandlersFactory(*factory, server); auto main_handler = std::make_unique>(server); main_handler->allowPostAndGetParamsRequest(); @@ -157,7 +155,7 @@ Poco::Net::HTTPRequestHandlerFactory * createHandlerFactory(IServer & server, As static const auto ping_response_expression = "Ok.\n"; static const auto root_response_expression = "config://http_server_default_response"; -void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics * async_metrics) +void addCommonDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server) { auto root_handler = std::make_unique>(server, root_response_expression); root_handler->attachStrictPath("/")->allowGetAndHeadRequest(); @@ -170,13 +168,22 @@ void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer auto replicas_status_handler = std::make_unique>(server); replicas_status_handler->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest(); factory.addHandler(replicas_status_handler.release()); +} + +void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics & async_metrics) +{ + addCommonDefaultHandlersFactory(factory, server); + + auto query_handler = std::make_unique>(server, "query"); + query_handler->allowPostAndGetParamsRequest(); + factory.addHandler(query_handler.release()); /// We check that prometheus handler will be served on current (default) port. - /// Otherwise it will be created separately, see below. - if (async_metrics && server.config().has("prometheus") && server.config().getInt("prometheus.port", 0) == 0) + /// Otherwise it will be created separately, see createHandlerFactory(...). + if (server.config().has("prometheus") && server.config().getInt("prometheus.port", 0) == 0) { auto prometheus_handler = std::make_unique>( - server, PrometheusMetricsWriter(server.config(), "prometheus", *async_metrics)); + server, PrometheusMetricsWriter(server.config(), "prometheus", async_metrics)); prometheus_handler->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics"))->allowGetAndHeadRequest(); factory.addHandler(prometheus_handler.release()); } diff --git a/src/Server/HTTPHandlerFactory.h b/src/Server/HTTPHandlerFactory.h index 8e21a13ba18..3e8313172eb 100644 --- a/src/Server/HTTPHandlerFactory.h +++ b/src/Server/HTTPHandlerFactory.h @@ -103,8 +103,6 @@ private: std::function creator; }; -void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics * async_metrics); - Poco::Net::HTTPRequestHandlerFactory * createStaticHandlerFactory(IServer & server, const std::string & config_prefix); Poco::Net::HTTPRequestHandlerFactory * createDynamicHandlerFactory(IServer & server, const std::string & config_prefix); diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index b31913ba962..b15cd1fdb89 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -124,6 +124,9 @@ def test_defaults_http_handlers(): assert 200 == cluster.instance.http_request('replicas_status', method='GET').status_code assert 'Ok.\n' == cluster.instance.http_request('replicas_status', method='GET').content + assert 200 == cluster.instance.http_request('?query=SELECT+1', method='GET').status_code + assert '1\n' == cluster.instance.http_request('?query=SELECT+1', method='GET').content + def test_prometheus_handler(): with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster: assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code From 10566e2b43705364fc1d54224a5393e681f16a5b Mon Sep 17 00:00:00 2001 From: Mikhail Malafeev <50805089+demo-99@users.noreply.github.com> Date: Mon, 15 Jun 2020 17:03:01 +0500 Subject: [PATCH 216/329] Remove duplicate ORDER BY and DISTINCT from subqueries (#10067) --- src/Core/Settings.h | 1 + src/Interpreters/DuplicateDistinctVisitor.h | 72 ++++++++++ src/Interpreters/DuplicateOrderByVisitor.h | 127 ++++++++++++++++++ src/Interpreters/SyntaxAnalyzer.cpp | 18 +++ .../duplicate_order_by_and_distinct.xml | 10 ++ ..._duplicate_order_by_and_distinct.reference | 14 ++ .../01305_duplicate_order_by_and_distinct.sql | 124 +++++++++++++++++ ...t_optimize_for_distributed_table.reference | 2 + ...istinct_optimize_for_distributed_table.sql | 20 +++ 9 files changed, 388 insertions(+) create mode 100644 src/Interpreters/DuplicateDistinctVisitor.h create mode 100644 src/Interpreters/DuplicateOrderByVisitor.h create mode 100644 tests/performance/duplicate_order_by_and_distinct.xml create mode 100644 tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference create mode 100644 tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.sql create mode 100644 tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.reference create mode 100644 tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 1e7728709ba..adc804c3a28 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -360,6 +360,7 @@ struct Settings : public SettingsCollection M(SettingBool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \ M(SettingUInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \ M(SettingBool, optimize_arithmetic_operations_in_aggregate_functions, true, "Move arithmetic operations out of aggregation functions", 0) \ + M(SettingBool, optimize_duplicate_order_by_and_distinct, true, "Remove duplicate ORDER BY and DISTINCT if it's possible", 0) \ M(SettingBool, optimize_if_chain_to_miltiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \ M(SettingBool, allow_experimental_alter_materialized_view_structure, false, "Allow atomic alter on Materialized views. Work in progress.", 0) \ M(SettingBool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there're constants there", 0) \ diff --git a/src/Interpreters/DuplicateDistinctVisitor.h b/src/Interpreters/DuplicateDistinctVisitor.h new file mode 100644 index 00000000000..9ce2624f5bd --- /dev/null +++ b/src/Interpreters/DuplicateDistinctVisitor.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +/// Removes duplicate DISTINCT from queries. +class DuplicateDistinctMatcher +{ +public: + struct Data + { + bool is_distinct; + std::vector last_ids; + }; + + static void visit(const ASTPtr & ast, Data & data) + { + auto * select_query = ast->as(); + if (select_query) + visit(*select_query, data); + } + + static void visit(ASTSelectQuery & select_query, Data & data) + { + if (!select_query.distinct || !select_query.select()) + return; + + /// Optimize shouldn't work for distributed tables + for (const auto & elem : select_query.children) + { + if (elem->as() && !elem->as()->is_standalone) + return; + } + + auto expression_list = select_query.select(); + std::vector current_ids; + + if (expression_list->children.empty()) + return; + + current_ids.reserve(expression_list->children.size()); + for (const auto & id : expression_list->children) + current_ids.push_back(id->getColumnName()); + + if (data.is_distinct && current_ids == data.last_ids) + select_query.distinct = false; + + data.is_distinct = true; + data.last_ids = std::move(current_ids); + } + + static bool needChildVisit(const ASTPtr &, const ASTPtr &) + { + return true; + } + +}; + +using DuplicateDistinctVisitor = InDepthNodeVisitor; + +} diff --git a/src/Interpreters/DuplicateOrderByVisitor.h b/src/Interpreters/DuplicateOrderByVisitor.h new file mode 100644 index 00000000000..85f34377e54 --- /dev/null +++ b/src/Interpreters/DuplicateOrderByVisitor.h @@ -0,0 +1,127 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +/// Checks if SELECT has stateful functions +class ASTFunctionStatefulData +{ +public: + using TypeToVisit = ASTFunction; + + const Context & context; + bool & is_stateful; + void visit(ASTFunction & ast_function, ASTPtr &) + { + if (ast_function.name == "any" || ast_function.name == "groupArray") + { + is_stateful = true; + return; + } + + const auto & function = FunctionFactory::instance().tryGet(ast_function.name, context); + + if (function && function->isStateful()) + { + is_stateful = true; + return; + } + } +}; + +using ASTFunctionStatefulMatcher = OneTypeMatcher; +using ASTFunctionStatefulVisitor = InDepthNodeVisitor; + + +/// Erases unnecessary ORDER BY from subquery +class DuplicateOrderByFromSubqueriesData +{ +public: + using TypeToVisit = ASTSelectQuery; + + bool done = false; + + void visit(ASTSelectQuery & select_query, ASTPtr &) + { + if (done) + return; + + if (select_query.orderBy() && !select_query.limitBy() && !select_query.limitByOffset() && + !select_query.limitByLength() && !select_query.limitLength() && !select_query.limitOffset()) + { + select_query.setExpression(ASTSelectQuery::Expression::ORDER_BY, nullptr); + } + + done = true; + } +}; + +using DuplicateOrderByFromSubqueriesMatcher = OneTypeMatcher; +using DuplicateOrderByFromSubqueriesVisitor = InDepthNodeVisitor; + + +/// Finds SELECT that can be optimized +class DuplicateOrderByData +{ +public: + using TypeToVisit = ASTSelectQuery; + + const Context & context; + bool done = false; + + void visit(ASTSelectQuery & select_query, ASTPtr &) + { + if (done) + return; + + /// Disable optimization for distributed tables + for (const auto & elem : select_query.children) + { + if (elem->as() && !elem->as()->is_standalone) + return; + } + + if (select_query.orderBy() || select_query.groupBy()) + { + for (auto & elem : select_query.children) + { + if (elem->as()) + { + bool is_stateful = false; + ASTFunctionStatefulVisitor::Data data{context, is_stateful}; + ASTFunctionStatefulVisitor(data).visit(elem); + if (is_stateful) + return; + } + } + + if (auto select_table_ptr = select_query.tables()) + { + if (auto * select_table = select_table_ptr->as()) + { + if (!select_table->children.empty()) + { + DuplicateOrderByFromSubqueriesVisitor::Data data{false}; + DuplicateOrderByFromSubqueriesVisitor(data).visit(select_table->children[0]); + } + } + } + } + } +}; + +using DuplicateOrderByMatcher = OneTypeMatcher; +using DuplicateOrderByVisitor = InDepthNodeVisitor; + +} diff --git a/src/Interpreters/SyntaxAnalyzer.cpp b/src/Interpreters/SyntaxAnalyzer.cpp index 8f6d368e6ad..4bfae18f9a5 100644 --- a/src/Interpreters/SyntaxAnalyzer.cpp +++ b/src/Interpreters/SyntaxAnalyzer.cpp @@ -23,12 +23,15 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include @@ -370,6 +373,18 @@ void optimizeOrderBy(const ASTSelectQuery * select_query) elems = std::move(unique_elems); } +/// Optimize duplicate ORDER BY and DISTINCT +void optimizeDuplicateOrderByAndDistinct(ASTPtr & query, bool optimize_duplicate_order_by_and_distinct, const Context & context) +{ + if (optimize_duplicate_order_by_and_distinct) + { + DuplicateOrderByVisitor::Data order_by_data{context, false}; + DuplicateOrderByVisitor(order_by_data).visit(query); + DuplicateDistinctVisitor::Data distinct_data{}; + DuplicateDistinctVisitor(distinct_data).visit(query); + } +} + /// Remove duplicate items from LIMIT BY. void optimizeLimitBy(const ASTSelectQuery * select_query) { @@ -831,6 +846,9 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( /// Remove duplicate items from ORDER BY. optimizeOrderBy(select_query); + /// Remove duplicate ORDER BY and DISTINCT from subqueries. + optimizeDuplicateOrderByAndDistinct(query, settings.optimize_duplicate_order_by_and_distinct, context); + /// Remove duplicated elements from LIMIT BY clause. optimizeLimitBy(select_query); diff --git a/tests/performance/duplicate_order_by_and_distinct.xml b/tests/performance/duplicate_order_by_and_distinct.xml new file mode 100644 index 00000000000..0c05af3fc56 --- /dev/null +++ b/tests/performance/duplicate_order_by_and_distinct.xml @@ -0,0 +1,10 @@ + + + hits_10m_single + + + SELECT * FROM (SELECT CounterID, EventDate FROM hits_10m_single ORDER BY CounterID DESC) ORDER BY EventDate, CounterID FORMAT Null + SELECT DISTINCT * FROM (SELECT DISTINCT CounterID, EventDate FROM hits_10m_single) FORMAT Null + SELECT DISTINCT * FROM (SELECT DISTINCT CounterID, EventDate FROM hits_10m_single ORDER BY CounterID DESC) ORDER BY toStartOfWeek(EventDate) FORMAT Null + + diff --git a/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference b/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference new file mode 100644 index 00000000000..208f3d1abe5 --- /dev/null +++ b/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference @@ -0,0 +1,14 @@ +SELECT number\nFROM \n(\n SELECT number\n FROM \n (\n SELECT DISTINCT number\n FROM numbers(3)\n )\n)\nORDER BY number ASC +0 +1 +2 +SELECT DISTINCT number\nFROM \n(\n SELECT DISTINCT number\n FROM \n (\n SELECT DISTINCT number\n FROM numbers(3)\n ORDER BY number ASC\n )\n ORDER BY number ASC\n)\nORDER BY number ASC +0 +1 +2 +SELECT number\nFROM \n(\n SELECT DISTINCT number\n FROM \n (\n SELECT DISTINCT number % 2 AS number\n FROM numbers(3)\n )\n)\nORDER BY number ASC +0 +1 +SELECT DISTINCT number\nFROM \n(\n SELECT DISTINCT number\n FROM \n (\n SELECT DISTINCT number % 2 AS number\n FROM numbers(3)\n ORDER BY number ASC\n )\n ORDER BY number ASC\n)\nORDER BY number ASC +0 +1 diff --git a/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.sql b/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.sql new file mode 100644 index 00000000000..a660e5f0b77 --- /dev/null +++ b/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.sql @@ -0,0 +1,124 @@ +set enable_debug_queries = 1; +set optimize_duplicate_order_by_and_distinct = 1; + +analyze SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +set optimize_duplicate_order_by_and_distinct = 0; + +analyze SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +set optimize_duplicate_order_by_and_distinct = 1; + +analyze SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT number % 2 + AS number + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT number % 2 + AS number + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +set optimize_duplicate_order_by_and_distinct = 0; + +analyze SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT number % 2 + AS number + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; + +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT number % 2 + AS number + FROM numbers(3) + ORDER BY number + ) + ORDER BY number +) +ORDER BY number; diff --git a/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.reference b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.reference new file mode 100644 index 00000000000..aa47d0d46d4 --- /dev/null +++ b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.reference @@ -0,0 +1,2 @@ +0 +0 diff --git a/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql new file mode 100644 index 00000000000..e1467bacf2f --- /dev/null +++ b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql @@ -0,0 +1,20 @@ +set optimize_duplicate_order_by_and_distinct = 1; +SELECT DISTINCT number +FROM +( + SELECT DISTINCT number + FROM remote('127.0.0.{1,2}', system.numbers) + LIMIT 1 + SETTINGS distributed_group_by_no_merge = 1 +); + +set optimize_duplicate_order_by_and_distinct = 0; +SELECT DISTINCT number +FROM +( + SELECT DISTINCT number + FROM remote('127.0.0.{1,2}', system.numbers) + LIMIT 1 + SETTINGS distributed_group_by_no_merge = 1 +); + From 24059efad5dadac02a728d0aedbc419e0a4b0e53 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Mon, 15 Jun 2020 15:36:10 +0300 Subject: [PATCH 217/329] Change push down logic in VIEW (#11513) --- src/Interpreters/CrossToInnerJoinVisitor.cpp | 4 +- src/Interpreters/IdentifierSemantic.cpp | 28 ++++--- src/Interpreters/IdentifierSemantic.h | 6 +- src/Interpreters/InterpreterExplainQuery.cpp | 61 ++++----------- src/Interpreters/InterpreterSelectQuery.cpp | 58 +++++++++----- src/Interpreters/InterpreterSelectQuery.h | 2 + .../JoinToSubqueryTransformVisitor.cpp | 3 +- src/Interpreters/JoinedTables.h | 5 +- src/Storages/SelectQueryInfo.h | 2 + src/Storages/StorageView.cpp | 78 +++++++++---------- src/Storages/StorageView.h | 8 +- ...76_predicate_optimizer_with_view.reference | 8 +- ...e_with_ambiguous_column_and_view.reference | 3 + ...rewrite_with_ambiguous_column_and_view.sql | 35 +++++++++ 14 files changed, 171 insertions(+), 130 deletions(-) create mode 100644 tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.reference create mode 100644 tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql diff --git a/src/Interpreters/CrossToInnerJoinVisitor.cpp b/src/Interpreters/CrossToInnerJoinVisitor.cpp index b2f3f56be4d..5ebebae2578 100644 --- a/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -202,11 +202,11 @@ private: { std::optional left_table_pos = IdentifierSemantic::getMembership(left); if (!left_table_pos) - left_table_pos = IdentifierSemantic::chooseTable(left, tables); + left_table_pos = IdentifierSemantic::chooseTableColumnMatch(left, tables); std::optional right_table_pos = IdentifierSemantic::getMembership(right); if (!right_table_pos) - right_table_pos = IdentifierSemantic::chooseTable(right, tables); + right_table_pos = IdentifierSemantic::chooseTableColumnMatch(right, tables); if (left_table_pos && right_table_pos && (*left_table_pos != *right_table_pos)) { diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index 8f254b50400..f661ec2ae71 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -16,7 +16,8 @@ namespace { template -std::optional tryChooseTable(const ASTIdentifier & identifier, const std::vector & tables, bool allow_ambiguous) +std::optional tryChooseTable(const ASTIdentifier & identifier, const std::vector & tables, + bool allow_ambiguous, bool column_match [[maybe_unused]] = false) { using ColumnMatch = IdentifierSemantic::ColumnMatch; @@ -27,6 +28,13 @@ std::optional tryChooseTable(const ASTIdentifier & identifier, const std for (size_t i = 0; i < tables.size(); ++i) { auto match = IdentifierSemantic::canReferColumnToTable(identifier, tables[i]); + + if constexpr (std::is_same_v) + { + if (column_match && match == ColumnMatch::NoMatch && identifier.isShort() && tables[i].hasColumn(identifier.shortName())) + match = ColumnMatch::ColumnName; + } + if (match != ColumnMatch::NoMatch) { if (match > best_match) @@ -125,12 +133,17 @@ std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & iden return tryChooseTable(identifier, tables, ambiguous); } -std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const std::vector & tables, - bool ambiguous) +std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const TablesWithColumns & tables, bool ambiguous) { return tryChooseTable(identifier, tables, ambiguous); } +std::optional IdentifierSemantic::chooseTableColumnMatch(const ASTIdentifier & identifier, const TablesWithColumns & tables, + bool ambiguous) +{ + return tryChooseTable(identifier, tables, ambiguous, true); +} + StorageID IdentifierSemantic::extractDatabaseAndTable(const ASTIdentifier & identifier) { if (identifier.name_parts.size() > 2) @@ -191,14 +204,9 @@ IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const } IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const ASTIdentifier & identifier, - const TableWithColumnNamesAndTypes & db_and_table) + const TableWithColumnNamesAndTypes & table_with_columns) { - ColumnMatch match = canReferColumnToTable(identifier, db_and_table.table); -#if 0 - if (match == ColumnMatch::NoMatch && identifier.isShort() && db_and_table.hasColumn(identifier.shortName())) - match = ColumnMatch::ColumnName; -#endif - return match; + return canReferColumnToTable(identifier, table_with_columns.table); } /// Strip qualificators from left side of column name. diff --git a/src/Interpreters/IdentifierSemantic.h b/src/Interpreters/IdentifierSemantic.h index 7e84e10a26f..0aef297c734 100644 --- a/src/Interpreters/IdentifierSemantic.h +++ b/src/Interpreters/IdentifierSemantic.h @@ -41,7 +41,7 @@ struct IdentifierSemantic static std::optional extractNestedName(const ASTIdentifier & identifier, const String & table_name); static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); - static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNamesAndTypes & db_and_table); + static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNamesAndTypes & table_with_columns); static void setColumnShortName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); static void setColumnLongName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); @@ -52,7 +52,9 @@ struct IdentifierSemantic static std::optional getMembership(const ASTIdentifier & identifier); static std::optional chooseTable(const ASTIdentifier &, const std::vector & tables, bool allow_ambiguous = false); - static std::optional chooseTable(const ASTIdentifier &, const std::vector & tables, + static std::optional chooseTable(const ASTIdentifier &, const TablesWithColumns & tables, + bool allow_ambiguous = false); + static std::optional chooseTableColumnMatch(const ASTIdentifier &, const TablesWithColumns & tables, bool allow_ambiguous = false); private: diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index dacd7ca5f20..4890287e81e 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -4,20 +4,15 @@ #include #include #include -#include -#include #include +#include #include #include -#include #include -#include #include #include -#include +#include -#include -#include #include #include @@ -31,56 +26,30 @@ namespace { struct Data { - bool analyzed = false; const Context & context; }; - static bool needChildVisit(ASTPtr &, ASTPtr &) { return true; } + static bool needChildVisit(ASTPtr & node, ASTPtr &) + { + return !node->as(); + } static void visit(ASTPtr & ast, Data & data) { - if (auto * select_query = ast->as()) - visit(*select_query, ast, data); - if (auto * union_select_query = ast->as()) - visit(*union_select_query, ast, data); + if (auto * select = ast->as()) + visit(*select, ast, data); } - static void visit(ASTSelectQuery & select_query, ASTPtr &, Data & data) + static void visit(ASTSelectQuery & select, ASTPtr & node, Data & data) { - if (!select_query.tables()) - return; + InterpreterSelectQuery interpreter( + node, data.context, SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze().modify()); - for (const auto & child : select_query.tables()->children) + const SelectQueryInfo & query_info = interpreter.getQueryInfo(); + if (query_info.view_query) { - auto * tables_element = child->as(); - - if (tables_element && tables_element->table_expression) - visit(*tables_element->table_expression->as(), select_query, data); - } - } - - static void visit(ASTSelectWithUnionQuery &, ASTPtr & node, Data & data) - { - if (!data.analyzed) - { - data.analyzed = true; - InterpreterSelectWithUnionQuery interpreter( - node, data.context, SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze().modify()); - } - } - - static void visit(ASTTableExpression & expression, ASTSelectQuery & select_query, Data & data) - { - if (data.context.getSettingsRef().enable_optimize_predicate_expression && expression.database_and_table_name) - { - if (const auto * identifier = expression.database_and_table_name->as()) - { - auto table_id = data.context.resolveStorageID(*identifier); - const auto & storage = DatabaseCatalog::instance().getTable(table_id, data.context); - - if (auto * storage_view = dynamic_cast(storage.get())) - storage_view->getRuntimeViewQuery(&select_query, data.context, true); - } + ASTPtr tmp; + StorageView::replaceWithSubquery(select, query_info.view_query->clone(), tmp); } } }; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 297679c4616..ac17a3042d8 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include @@ -186,6 +187,26 @@ static Context getSubqueryContext(const Context & context) return subquery_context; } +static void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & tables, const String & database, const Settings & settings) +{ + ASTSelectQuery & select = query->as(); + + Aliases aliases; + if (ASTPtr with = select.with()) + QueryAliasesNoSubqueriesVisitor(aliases).visit(with); + QueryAliasesNoSubqueriesVisitor(aliases).visit(select.select()); + + CrossToInnerJoinVisitor::Data cross_to_inner{tables, aliases, database}; + CrossToInnerJoinVisitor(cross_to_inner).visit(query); + + size_t rewriter_version = settings.multiple_joins_rewriter_version; + if (!rewriter_version || rewriter_version > 2) + throw Exception("Bad multiple_joins_rewriter_version setting value: " + settings.multiple_joins_rewriter_version.toString(), + ErrorCodes::INVALID_SETTING_VALUE); + JoinToSubqueryTransformVisitor::Data join_to_subs_data{tables, aliases, rewriter_version}; + JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query); +} + InterpreterSelectQuery::InterpreterSelectQuery( const ASTPtr & query_ptr_, const Context & context_, @@ -242,29 +263,14 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// Rewrite JOINs if (!has_input && joined_tables.tablesCount() > 1) { - ASTSelectQuery & select = getSelectQuery(); + rewriteMultipleJoins(query_ptr, joined_tables.tablesWithColumns(), context->getCurrentDatabase(), settings); - Aliases aliases; - if (ASTPtr with = select.with()) - QueryAliasesNoSubqueriesVisitor(aliases).visit(with); - QueryAliasesNoSubqueriesVisitor(aliases).visit(select.select()); - - CrossToInnerJoinVisitor::Data cross_to_inner{joined_tables.tablesWithColumns(), aliases, context->getCurrentDatabase()}; - CrossToInnerJoinVisitor(cross_to_inner).visit(query_ptr); - - size_t rewriter_version = settings.multiple_joins_rewriter_version; - if (!rewriter_version || rewriter_version > 2) - throw Exception("Bad multiple_joins_rewriter_version setting value: " + settings.multiple_joins_rewriter_version.toString(), - ErrorCodes::INVALID_SETTING_VALUE); - JoinToSubqueryTransformVisitor::Data join_to_subs_data{joined_tables.tablesWithColumns(), aliases, rewriter_version}; - JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query_ptr); - - joined_tables.reset(select); + joined_tables.reset(getSelectQuery()); joined_tables.resolveTables(); if (storage && joined_tables.isLeftTableSubquery()) { - /// Rewritten with subquery. Free storage here locks here. + /// Rewritten with subquery. Free storage locks here. storage = {}; table_lock.release(); table_id = StorageID::createEmpty(); @@ -288,12 +294,28 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) row_policy_filter = context->getRowPolicyCondition(table_id.getDatabaseName(), table_id.getTableName(), RowPolicy::SELECT_FILTER); + StorageView * view = nullptr; + if (storage) + view = dynamic_cast(storage.get()); + auto analyze = [&] (bool try_move_to_prewhere) { + /// Allow push down and other optimizations for VIEW: replace with subquery and rewrite it. + ASTPtr view_table; + if (view) + view->replaceWithSubquery(getSelectQuery(), view_table); + syntax_analyzer_result = SyntaxAnalyzer(*context).analyzeSelect( query_ptr, SyntaxAnalyzerResult(source_header.getNamesAndTypesList(), storage), options, joined_tables.tablesWithColumns(), required_result_column_names, table_join); + if (view) + { + /// Restore original view name. Save rewritten subquery for future usage in StorageView. + query_info.view_query = view->restoreViewName(getSelectQuery(), view_table); + view = nullptr; + } + if (try_move_to_prewhere && storage && !row_policy_filter && query.where() && !query.prewhere() && !query.final()) { /// PREWHERE optimization: transfer some condition from WHERE to PREWHERE if enabled and viable diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index c60451d5f4a..8ed775f60ae 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -88,6 +88,8 @@ public: size_t getMaxStreams() const { return max_streams; } + const SelectQueryInfo & getQueryInfo() const { return query_info; } + private: InterpreterSelectQuery( const ASTPtr & query_ptr_, diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 7ed1bb9d1bb..9bfa9e1e98b 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -585,8 +585,9 @@ std::vector normalizeColumnNamesExtractNeeded( for (ASTIdentifier * ident : identifiers) { bool got_alias = aliases.count(ident->name); + bool allow_ambiguous = got_alias; /// allow ambiguous column overridden by an alias - if (auto table_pos = IdentifierSemantic::chooseTable(*ident, tables)) + if (auto table_pos = IdentifierSemantic::chooseTableColumnMatch(*ident, tables, allow_ambiguous)) { if (!ident->isShort()) { diff --git a/src/Interpreters/JoinedTables.h b/src/Interpreters/JoinedTables.h index 55244e1225c..2591b49527b 100644 --- a/src/Interpreters/JoinedTables.h +++ b/src/Interpreters/JoinedTables.h @@ -34,7 +34,8 @@ public: void makeFakeTable(StoragePtr storage, const Block & source_header); std::shared_ptr makeTableJoin(const ASTSelectQuery & select_query); - const std::vector & tablesWithColumns() const { return tables_with_columns; } + const TablesWithColumns & tablesWithColumns() const { return tables_with_columns; } + TablesWithColumns moveTablesWithColumns() { return std::move(tables_with_columns); } bool isLeftTableSubquery() const; bool isLeftTableFunction() const; @@ -49,7 +50,7 @@ public: private: Context context; std::vector table_expressions; - std::vector tables_with_columns; + TablesWithColumns tables_with_columns; /// Legacy (duplicated left table values) ASTPtr left_table_expression; diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index c4cd1035ea7..26b318f107b 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -70,6 +71,7 @@ using ReadInOrderOptimizerPtr = std::shared_ptr; struct SelectQueryInfo { ASTPtr query; + ASTPtr view_query; /// Optimized VIEW query SyntaxAnalyzerResultPtr syntax_analyzer_result; diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 97403a359c3..745fc823703 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -1,8 +1,6 @@ #include #include -#include #include -#include #include #include @@ -30,7 +28,6 @@ namespace ErrorCodes { extern const int INCORRECT_QUERY; extern const int LOGICAL_ERROR; - extern const int ALIAS_REQUIRED; } @@ -60,9 +57,12 @@ Pipes StorageView::read( Pipes pipes; ASTPtr current_inner_query = inner_query; - - if (context.getSettings().enable_optimize_predicate_expression) - current_inner_query = getRuntimeViewQuery(*query_info.query->as(), context); + if (query_info.view_query) + { + if (!query_info.view_query->as()) + throw Exception("Unexpected optimized VIEW query", ErrorCodes::LOGICAL_ERROR); + current_inner_query = query_info.view_query->clone(); + } InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names); @@ -87,60 +87,52 @@ Pipes StorageView::read( return pipes; } -ASTPtr StorageView::getRuntimeViewQuery(const ASTSelectQuery & outer_query, const Context & context) +static ASTTableExpression * getFirstTableExpression(ASTSelectQuery & select_query) { - auto temp_outer_query = outer_query.clone(); - auto * new_outer_select = temp_outer_query->as(); - return getRuntimeViewQuery(new_outer_select, context, false); -} - - -static void replaceTableNameWithSubquery(ASTSelectQuery * select_query, ASTPtr & subquery) -{ - auto * select_element = select_query->tables()->children[0]->as(); + auto * select_element = select_query.tables()->children[0]->as(); if (!select_element->table_expression) throw Exception("Logical error: incorrect table expression", ErrorCodes::LOGICAL_ERROR); - auto * table_expression = select_element->table_expression->as(); + return select_element->table_expression->as(); +} + +void StorageView::replaceWithSubquery(ASTSelectQuery & outer_query, ASTPtr view_query, ASTPtr & view_name) +{ + ASTTableExpression * table_expression = getFirstTableExpression(outer_query); if (!table_expression->database_and_table_name) throw Exception("Logical error: incorrect table expression", ErrorCodes::LOGICAL_ERROR); - const auto alias = table_expression->database_and_table_name->tryGetAlias(); + DatabaseAndTableWithAlias db_table(table_expression->database_and_table_name); + String alias = db_table.alias.empty() ? db_table.table : db_table.alias; + + view_name = table_expression->database_and_table_name; table_expression->database_and_table_name = {}; table_expression->subquery = std::make_shared(); - table_expression->subquery->children.push_back(subquery); - table_expression->children.push_back(table_expression->subquery); - if (!alias.empty()) - table_expression->subquery->setAlias(alias); + table_expression->subquery->children.push_back(view_query); + table_expression->subquery->setAlias(alias); + + for (auto & child : table_expression->children) + if (child.get() == view_name.get()) + child = view_query; } - -ASTPtr StorageView::getRuntimeViewQuery(ASTSelectQuery * outer_query, const Context & context, bool normalize) +ASTPtr StorageView::restoreViewName(ASTSelectQuery & select_query, const ASTPtr & view_name) { - auto runtime_view_query = inner_query->clone(); + ASTTableExpression * table_expression = getFirstTableExpression(select_query); - /// TODO: remove getTableExpressions and getTablesWithColumns - { - const auto & table_expressions = getTableExpressions(*outer_query); - const auto & tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); + if (!table_expression->subquery) + throw Exception("Logical error: incorrect table expression", ErrorCodes::LOGICAL_ERROR); - replaceTableNameWithSubquery(outer_query, runtime_view_query); - if (context.getSettingsRef().joined_subquery_requires_alias && tables_with_columns.size() > 1) - { - for (const auto & pr : tables_with_columns) - if (pr.table.table.empty() && pr.table.alias.empty()) - throw Exception("Not unique subquery in FROM requires an alias (or joined_subquery_requires_alias=0 to disable restriction).", - ErrorCodes::ALIAS_REQUIRED); - } + ASTPtr subquery = table_expression->subquery; + table_expression->subquery = {}; + table_expression->database_and_table_name = view_name; - if (PredicateExpressionsOptimizer(context, tables_with_columns, context.getSettings()).optimize(*outer_query) && normalize) - InterpreterSelectWithUnionQuery( - runtime_view_query, context, SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze().modify(), {}); - } - - return runtime_view_query; + for (auto & child : table_expression->children) + if (child.get() == subquery.get()) + child = view_name; + return subquery->children[0]; } void registerStorageView(StorageFactory & factory) diff --git a/src/Storages/StorageView.h b/src/Storages/StorageView.h index 86550db83ce..61885460249 100644 --- a/src/Storages/StorageView.h +++ b/src/Storages/StorageView.h @@ -29,9 +29,13 @@ public: size_t max_block_size, unsigned num_streams) override; - ASTPtr getRuntimeViewQuery(const ASTSelectQuery & outer_query, const Context & context); + void replaceWithSubquery(ASTSelectQuery & select_query, ASTPtr & view_name) const + { + replaceWithSubquery(select_query, inner_query->clone(), view_name); + } - ASTPtr getRuntimeViewQuery(ASTSelectQuery * outer_query, const Context & context, bool normalize); + static void replaceWithSubquery(ASTSelectQuery & outer_query, ASTPtr view_query, ASTPtr & view_name); + static ASTPtr restoreViewName(ASTSelectQuery & select_query, const ASTPtr & view_name); private: ASTPtr inner_query; diff --git a/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference b/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference index 1e92e7b8596..c9b4ed1e1f7 100644 --- a/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference +++ b/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference @@ -1,4 +1,4 @@ -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 1\n)\nWHERE id = 1 -SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 2\n)\nWHERE id = 2 -SELECT id\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 1\n)\nWHERE id = 1 -SELECT id\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 1\n) AS s\nWHERE id = 1 +SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT *\n FROM default.test\n HAVING id = 1\n) AS test_view\nWHERE id = 1 +SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT *\n FROM default.test\n HAVING id = 2\n) AS test_view\nWHERE id = 2 +SELECT id\nFROM \n(\n SELECT *\n FROM default.test\n HAVING id = 1\n) AS test_view\nWHERE id = 1 +SELECT id\nFROM \n(\n SELECT *\n FROM default.test\n HAVING id = 1\n) AS s\nWHERE id = 1 diff --git a/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.reference b/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.reference new file mode 100644 index 00000000000..461a50ea880 --- /dev/null +++ b/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.reference @@ -0,0 +1,3 @@ +1 1 1 +2 2 0 +1 val11 val21 val31 diff --git a/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql b/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql new file mode 100644 index 00000000000..c90d01ff76d --- /dev/null +++ b/tests/queries/0_stateless/01144_join_rewrite_with_ambiguous_column_and_view.sql @@ -0,0 +1,35 @@ +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS view1; + +CREATE TABLE t1 (id UInt32, value1 String) ENGINE MergeTree() ORDER BY id; +CREATE TABLE t2 (id UInt32, value2 String) ENGINE MergeTree() ORDER BY id; +CREATE TABLE t3 (id UInt32, value3 String) ENGINE MergeTree() ORDER BY id; + +INSERT INTO t1 (id, value1) VALUES (1, 'val11'); +INSERT INTO t2 (id, value2) VALUES (1, 'val21'); +INSERT INTO t3 (id, value3) VALUES (1, 'val31'); + +SET multiple_joins_rewriter_version = 2; +SET enable_optimize_predicate_expression = 1; + +SELECT t1.id, t2.id as id, t3.id as value +FROM (select number as id, 42 as value from numbers(4)) t1 +LEFT JOIN (select number as id, 42 as value from numbers(3)) t2 ON t1.id = t2.id +LEFT JOIN (select number as id, 42 as value from numbers(2)) t3 ON t1.id = t3.id +WHERE id > 0 AND value < 42; + +CREATE VIEW IF NOT EXISTS view1 AS + SELECT t1.id AS id, t1.value1 AS value1, t2.value2 AS value2, t3.value3 AS value3 + FROM t1 + LEFT JOIN t2 ON t1.id = t2.id + LEFT JOIN t3 ON t1.id = t3.id + WHERE t1.id > 0; + +SELECT * FROM view1 WHERE id = 1; + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS view1; From c7140724a8c2abbd7793744904cf475841d927fa Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 15 Jun 2020 16:25:27 +0300 Subject: [PATCH 218/329] Fix that ALTER USER RENAME could change allowed hosts. --- src/Parsers/ParserCreateUserQuery.cpp | 16 +++++----------- .../01075_allowed_client_hosts.reference | 4 ++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index 3bf7e508220..e03f8334d42 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -23,14 +23,14 @@ namespace ErrorCodes namespace { - bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_name, std::optional & new_host_pattern) + bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_name) { return IParserBase::wrapParseImpl(pos, [&] { if (!ParserKeyword{"RENAME TO"}.ignore(pos, expected)) return false; - return parseUserName(pos, expected, new_name, new_host_pattern); + return parseUserName(pos, expected, new_name); }); } @@ -274,7 +274,6 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec return false; String new_name; - std::optional new_host_pattern; std::optional authentication; std::optional hosts; std::optional add_hosts; @@ -302,7 +301,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (alter) { - if (new_name.empty() && parseRenameTo(pos, expected, new_name, new_host_pattern)) + if (new_name.empty() && parseRenameTo(pos, expected, new_name)) continue; if (parseHosts(pos, expected, "ADD", add_hosts) || parseHosts(pos, expected, "DROP", remove_hosts)) @@ -312,13 +311,8 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec break; } - if (!hosts) - { - if (!alter && host_pattern) - hosts.emplace().addLikePattern(*host_pattern); - else if (alter && new_host_pattern) - hosts.emplace().addLikePattern(*new_host_pattern); - } + if (!alter && !hosts && host_pattern) + hosts.emplace().addLikePattern(*host_pattern); auto query = std::make_shared(); node = query; diff --git a/tests/queries/0_stateless/01075_allowed_client_hosts.reference b/tests/queries/0_stateless/01075_allowed_client_hosts.reference index 3fdea9d1cda..5fb11bae65e 100644 --- a/tests/queries/0_stateless/01075_allowed_client_hosts.reference +++ b/tests/queries/0_stateless/01075_allowed_client_hosts.reference @@ -13,5 +13,5 @@ CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite\\\\.com\', \'.*\\\\ CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite2\\\\.com\', \'.*\\\\.anothersite2\\\\.org\' CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite3\\\\.com\', \'.*\\\\.anothersite3\\\\.org\' CREATE USER `test_user_01075_x@localhost` HOST LOCAL -CREATE USER test_user_01075_x -CREATE USER `test_user_01075_x@192.168.23.15` HOST LIKE \'192.168.23.15\' +CREATE USER test_user_01075_x HOST LOCAL +CREATE USER `test_user_01075_x@192.168.23.15` HOST LOCAL From 8945d0073a9532c8a27264d7d37310f7acb28570 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Jun 2020 16:37:40 +0300 Subject: [PATCH 219/329] Fix misunderstanding bug in mutations finalization --- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 28 +++++---- .../MergeTree/ReplicatedMergeTreeQueue.h | 11 +++- ..._long_failing_mutation_zookeeper.reference | 6 ++ .../01318_long_failing_mutation_zookeeper.sh | 57 +++++++++++++++++++ 4 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.reference create mode 100755 tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.sh diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 8a9dbceba04..7cb46ffda7d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -201,21 +201,28 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( if (is_successful) { - if (!entry->actual_new_part_name.empty()) { /// We don't add bigger fetched part to current_parts because we /// have an invariant `virtual_parts` = `current_parts` + `queue`. - /// But we can remove it from mutations, because we actually have it. - removePartFromMutations(entry->actual_new_part_name); + /// + /// But we remove covered parts from mutations, because we actually + /// have replacing part. + Strings covered_parts = current_parts.getPartsCoveredBy(MergeTreePartInfo::fromPartName(entry->actual_new_part_name, format_version)); + + for (const auto & covered_part : covered_parts) + removePartFromMutations(covered_part); } for (const String & virtual_part_name : entry->getVirtualPartNames()) { - current_parts.add(virtual_part_name); - /// Each processed part may be already mutated, so we try to remove - /// all current parts from mutations. - removePartFromMutations(virtual_part_name); + Strings replaced_parts; + current_parts.add(virtual_part_name, &replaced_parts); + + /// These parts are already covered by newer part, we don't have to + /// mutate it. + for (const auto & replaced_part : replaced_parts) + removePartFromMutations(replaced_part); } String drop_range_part_name; @@ -240,11 +247,11 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( { for (const String & virtual_part_name : entry->getVirtualPartNames()) { - /// Because execution of the entry is unsuccessful, `virtual_part_name` will never appear - /// so we won't need to mutate it. + /// Because execution of the entry is unsuccessful, + /// `virtual_part_name` will never appear so we won't need to mutate + /// it. removePartFromMutations(virtual_part_name); } - } } @@ -678,6 +685,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C const String & partition_id = pair.first; Int64 block_num = pair.second; mutations_by_partition[partition_id].emplace(block_num, &mutation); + LOG_TRACE(log, "Adding mutation {} for partition {} for all block numbers less than {}", entry->znode_name, partition_id, block_num); } /// Initialize `mutation.parts_to_do`. First we need to mutate all parts in `current_parts`. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index e093e193381..8f3a7719076 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -107,8 +107,13 @@ private: ReplicatedMergeTreeMutationEntryPtr entry; - /// Parts we have to mutate to complete mutation. We use ActiveDataPartSet structure - /// to be able to manage covering and covered parts. + /// Current parts we have to mutate to complete mutation. + /// + /// current_part_name =mutation> result_part_name + /// ^~~parts_to_do~~^ ^~virtual_parts~^ + /// + /// We use ActiveDataPartSet structure to be able to manage covering and + /// covered parts. ActiveDataPartSet parts_to_do; /// Note that is_done is not equivalent to parts_to_do.size() == 0 @@ -204,7 +209,7 @@ private: /// Add part for mutations with block_number > part.getDataVersion() void addPartToMutations(const String & part_name); - /// Remove part from mutations which were assigned to mutate it + /// Remove part from mutations (parts_to_do) which were assigned to mutate it /// with block_number > part.getDataVersion() /// and block_number == part.getDataVersion() /// ^ (this may happen if we downloaded mutated part from other replica) diff --git a/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.reference b/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.reference new file mode 100644 index 00000000000..785123ae030 --- /dev/null +++ b/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.reference @@ -0,0 +1,6 @@ +90001 +2 +waiting default mutation_table 0000000000 MODIFY COLUMN `value` UInt64 +is_done parts_to_do +0 1 +MUTATE_PART 0_0_0_0_2 diff --git a/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.sh b/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.sh new file mode 100755 index 00000000000..3dc8b34fff6 --- /dev/null +++ b/tests/queries/0_stateless/01318_long_failing_mutation_zookeeper.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS mutation_table" + +$CLICKHOUSE_CLIENT --query " + CREATE TABLE mutation_table( + key UInt64, + value String + ) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/mutation_table', '1') + ORDER BY key + PARTITION BY key % 10 +" + +$CLICKHOUSE_CLIENT --query "INSERT INTO mutation_table select number, toString(number) from numbers(100000) where number % 10 != 0" + +$CLICKHOUSE_CLIENT --query "INSERT INTO mutation_table VALUES(0, 'hello')" + +$CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM mutation_table" + +$CLICKHOUSE_CLIENT --query "ALTER TABLE mutation_table MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=0" + +first_mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations where table='mutation_table' and database='$CLICKHOUSE_DATABASE'") + +# Here we have long sleeps, but they shouldn't lead to flaps. We just check that +# background mutation finalization function will be triggered at least once. In +# rare cases this test doesn't check anything, but will report OK. +sleep 7 + +$CLICKHOUSE_CLIENT --query "ALTER TABLE mutation_table MODIFY COLUMN value UInt32 SETTINGS replication_alter_partitions_sync=0" + + +#### just check that both mutations started +check_query="SELECT count() FROM system.mutations WHERE table='mutation_table' and database='$CLICKHOUSE_DATABASE'" + +query_result=`$CLICKHOUSE_CLIENT --query="$check_query" 2>&1` + +while [ "$query_result" != "2" ] +do + query_result=`$CLICKHOUSE_CLIENT --query="$check_query" 2>&1` + sleep 0.5 +done + +echo $query_result + +$CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$first_mutation_id'" + +sleep 7 + +$CLICKHOUSE_CLIENT --query "SELECT is_done, parts_to_do FROM system.mutations where table='mutation_table' and database='$CLICKHOUSE_DATABASE' FORMAT TSVWithNames" + +$CLICKHOUSE_CLIENT --query "SELECT type, new_part_name FROM system.replication_queue WHERE table='mutation_table' and database='$CLICKHOUSE_DATABASE'" + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS mutation_table" From 4c100dcf8f56909a739f2afbd626a353b13819b1 Mon Sep 17 00:00:00 2001 From: Tom Bombadil <565258751@qq.com> Date: Mon, 15 Jun 2020 22:10:23 +0800 Subject: [PATCH 220/329] Update syntax.md (#11679) translate to chinese doc --- docs/zh/sql-reference/syntax.md | 182 +++++++++++++++++--------------- 1 file changed, 95 insertions(+), 87 deletions(-) diff --git a/docs/zh/sql-reference/syntax.md b/docs/zh/sql-reference/syntax.md index b0aa9e7364f..687638a9be6 100644 --- a/docs/zh/sql-reference/syntax.md +++ b/docs/zh/sql-reference/syntax.md @@ -1,156 +1,162 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 31 -toc_title: "\u8BED\u6CD5" +toc_title: SQL语法 --- -# 语法 {#syntax} - -系统中有两种类型的解析器：完整SQL解析器（递归下降解析器）和数据格式解析器（快速流解析器）。 -在所有情况下，除了 `INSERT` 查询时，只使用完整的SQL解析器。 -该 `INSERT` 查询使用两个解析器: +# SQL语法 {#syntax} +CH有2类解析器：完整SQL解析器（递归式解析器），以及数据格式解析器（快速流式解析器） +除了 `INSERT` 查询，其它情况下仅使用完整SQL解析器。 + `INSERT`查询会同时使用2种解析器： ``` sql INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') ``` -该 `INSERT INTO t VALUES` 片段由完整的解析器解析，并且数据 `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` 由快速流解析器解析。 您也可以通过使用 [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions) 设置。 当 `input_format_values_interpret_expressions = 1`，ClickHouse首先尝试使用fast stream解析器解析值。 如果失败，ClickHouse将尝试对数据使用完整的解析器，将其视为SQL [表达式](#syntax-expressions). +含`INSERT INTO t VALUES` 的部分由完整SQL解析器处理，包含数据的部分 `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` 交给快速流式解析器解析。通过设置参数 [input\_format\_values\_interpret\_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions)，你也可以对数据部分开启完整SQL解析器。当 `input_format_values_interpret_expressions = 1` 时，CH优先采用快速流式解析器来解析数据。如果失败，CH再尝试用完整SQL解析器来处理，就像处理SQL [expression](#syntax-expressions) 一样。 -数据可以有任何格式。 当接收到查询时，服务器计算不超过 [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) RAM中请求的字节（默认为1MB），其余的是流解析。 -它允许避免与大的问题 `INSERT` 查询。 +数据可以采用任何格式。当CH接受到请求时，服务端先在内存中计算不超过 [max\_query\_size](../operations/settings/settings.md#settings-max_query_size) 字节的请求数据（默认1 mb），然后剩下部分交给快速流式解析器。 -使用时 `Values` 格式为 `INSERT` 查询，它可能看起来数据被解析相同的表达式 `SELECT` 查询，但事实并非如此。 该 `Values` 格式更为有限。 +这将避免在处理大型的 `INSERT`语句时出现问题。 -本文的其余部分将介绍完整的解析器。 有关格式解析器的详细信息，请参阅 [格式](../interfaces/formats.md) 科。 +当 `INSERT` 语句中使用 `Values` 形式时，看起来 数据部分的解析和解析`SELECT` 中的表达式相同，但并不是这样的。 `Values` 形式非常有限。 +该篇的剩余部分涵盖了完整SQL解析器。关于格式解析的更多信息，参见 [Formats](../interfaces/formats.md) 章节。 -## 空间 {#spaces} +## 空字符 {#spaces} -语法结构之间可能有任意数量的空格符号（包括查询的开始和结束）。 空格符号包括空格、制表符、换行符、CR和换页符。 +sql语句中（包含sql的起始和结束）可以有任意的空字符，这些空字符类型包括：空格字符，tab制表符，换行符，CR符，换页符等。 -## 评论 {#comments} +## 注释 {#comments} -ClickHouse支持SQL风格和C风格的注释。 -SQL风格的注释以下开头 `--` 并继续到线的末尾，一个空格后 `--` 可以省略。 -C型是从 `/*` 到 `*/`并且可以是多行，也不需要空格。 +CH支持SQL风格或C语言风格的注释： +- SQL风格的注释以 `--` 开始，直到行末，`--` 后紧跟的空格可以忽略 +- C语言风格的注释以 `/*` 开始，以 `*/` 结束，支持多行形式，同样可以省略 `/*` 后的空格 -## 关键词 {#syntax-keywords} +## 关键字 {#syntax-keywords} -当关键字对应于以下关键字时，不区分大小写: +以下场景的关键字是大小写不敏感的： +- 标准SQL。例如，`SELECT`, `select` 和 `SeLeCt` 都是允许的 +- 在某些流行的RDBMS中被实现的关键字，例如，`DateTime` 和 `datetime`是一样的 -- SQL标准。 例如, `SELECT`, `select` 和 `SeLeCt` 都是有效的。 -- 在一些流行的DBMS（MySQL或Postgres）中实现。 例如, `DateTime` 是一样的 `datetime`. -数据类型名称是否区分大小写可以在 `system.data_type_families` 桌子 +你可以在系统表 [system.data_type_families](../operations/system-tables.md#system_tables-data_type_families) 中检查某个数据类型的名称是否是大小写敏感型。 -与标准SQL相比，所有其他关键字（包括函数名称）都是 **区分大小写**. +和标准SQL相反，所有其它的关键字都是 **大小写敏感的**，包括函数名称。 +In contrast to standard SQL, all other keywords (including functions names) are **case-sensitive**. -不保留关键字；它们仅在相应的上下文中被视为保留关键字。 如果您使用 [标识符](#syntax-identifiers) 使用与关键字相同的名称，将它们括在双引号或反引号中。 例如，查询 `SELECT "FROM" FROM table_name` 是有效的，如果表 `table_name` 具有名称的列 `"FROM"`. +关键字不是保留的；它们仅在相应的上下文中才会被处理。如果你使用和关键字同名的 [变量名](#syntax-identifiers) ，需要使用双引号或转移符将它们包含起来。例如：如果表 `table_name` 包含列 `"FROM"`，那么 `SELECT "FROM" FROM table_name` 是合法的 -## 标识符 {#syntax-identifiers} +## 变量名 {#syntax-identifiers} -标识符是: +变量包括： +Identifiers are: -- 集群、数据库、表、分区和列名称。 -- 功能。 -- 数据类型。 -- [表达式别名](#syntax-expression_aliases). +- 集群，数据库，表，分区，列名称 +- 函数 +- 数据类型 +- 表达式别名 -标识符可以是引号或非引号。 后者是优选的。 +变量名可以使用反引号包含起来 -非引号标识符必须与正则表达式匹配 `^[a-zA-Z_][0-9a-zA-Z_]*$` 并且不能等于 [关键词](#syntax-keywords). 例: `x, _1, X_y__Z123_.` +没有使用反引号包含的变量名，必须匹配正则表达式 `^[a-zA-Z_][0-9a-zA-Z_]*$`，并且不能和 [关键字]相同 -如果要使用与关键字相同的标识符，或者要在标识符中使用其他符号，请使用双引号或反引号对其进行引用，例如, `"id"`, `` `id` ``. +如果想使用和关键字同名的变量名称，或者在变量名称中包含其它符号，你需要通过双引号或转义符号，例如： `"id"`, `` `id` `` -## 文字数 {#literals} +## 字符 {#literals} -有数字，字符串，复合和 `NULL` 文字。 +CH包含数字，字母，括号，NULL值等字符 ### 数字 {#numeric} -数值文字尝试进行分析: +数字类型字符会被做如下解析： +- 首先，当做64位的有符号整数，使用该函数 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) +- 如果失败，解析成64位无符号整数，同样使用函数 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) -- 首先，作为一个64位有符号的数字，使用 [strtoull](https://en.cppreference.com/w/cpp/string/byte/strtoul) 功能。 -- 如果不成功，作为64位无符号数，使用 [strtoll](https://en.cppreference.com/w/cpp/string/byte/strtol) 功能。 -- 如果不成功，作为一个浮点数使用 [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) 功能。 -- 否则，将返回错误。 +- 如果还失败了，试图解析成浮点型数值，使用函数 [strtod](https://en.cppreference.com/w/cpp/string/byte/strtof) +Numeric literal tries to be parsed: -文本值具有该值适合的最小类型。 -例如，1被解析为 `UInt8`，但256被解析为 `UInt16`. 有关详细信息，请参阅 [数据类型](../sql-reference/data-types/index.md). +- 最后，以上情形都不符合时，返回异常 -例: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. -### 字符串 {#syntax-string-literal} +数字类型的值类型为能容纳该值的最小数据类型。 +例如：1 解析成 `UInt8`型，256 则解析成 `UInt16`。更多信息，参见 [数据类型](../sql-reference/data-types/index.md) -仅支持单引号中的字符串文字。 封闭的字符可以反斜杠转义。 以下转义序列具有相应的特殊值: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. 在所有其他情况下，转义序列的格式为 `\c`，哪里 `c` 是任何字符，被转换为 `c`. 这意味着你可以使用序列 `\'`和`\\`. 该值将具有 [字符串](../sql-reference/data-types/string.md) 类型。 +例如: `1`, `18446744073709551615`, `0xDEADBEEF`, `01`, `0.1`, `1e100`, `-1e-100`, `inf`, `nan`. -在字符串文字中，你至少需要转义 `'` 和 `\`. 单引号可以用单引号，文字转义 `'It\'s'` 和 `'It''s'` 是平等的。 +### 字母 {#syntax-string-literal} +CH只支持用单引号包含的字母。特殊字符可通过反斜杠进行转义。下列转义字符都有相应的实际值： `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`。其它情况下，以 `\c`形式出现的转义字符，当`c`表示任意字符时，转义字符会转换成`c`。这意味着你可以使用 `\'`和`\\`。该值将拥有[String](../sql-reference/data-types/string.md)类型。 -### 化合物 {#compound} -数组使用方括号构造 `[1, 2, 3]`. Nuples用圆括号构造 `(1, 'Hello, world!', 2)`. -从技术上讲，这些不是文字，而是分别具有数组创建运算符和元组创建运算符的表达式。 -数组必须至少包含一个项目，元组必须至少包含两个项目。 -有一个单独的情况下，当元组出现在 `IN` a条款 `SELECT` 查询。 查询结果可以包含元组，但元组不能保存到数据库（除了具有以下内容的表 [记忆](../engines/table-engines/special/memory.md) 发动机）。 +在字符串中，你至少需要对 `'` 和 `\` 进行转义。单引号可以使用单引号转义，例如 `'It\'s'` 和 `'It''s'` 是相同的。 -### NULL {#null-literal} +### 括号 {#compound} +数组都是使用方括号进行构造 `[1, 2, 3]`，元组则使用圆括号 `(1, 'Hello, world!', 2)` -指示该值丢失。 +从技术上来讲，这些都不是字符串，而是包含创建数组和元组运算符的表达式。 -为了存储 `NULL` 在表字段中，它必须是 [可为空](../sql-reference/data-types/nullable.md) 类型。 +创建一个数组必须至少包含一个元素，创建一个元组至少包含2个元素 -根据数据格式（输入或输出), `NULL` 可能有不同的表示。 有关详细信息，请参阅以下文档 [数据格式](../interfaces/formats.md#formats). +当元组出现在 `SELECT` 查询的 `IN` 部分时，是一种例外情形。查询结果可以包含元组，但是元组类型不能保存到数据库中（除非表采用 [内存表](../engines/table-engines/special/memory.md)引擎） -处理有许多细微差别 `NULL`. 例如，如果比较操作的至少一个参数是 `NULL`，此操作的结果也是 `NULL`. 对于乘法，加法和其他操作也是如此。 有关详细信息，请阅读每个操作的文档。 -在查询中，您可以检查 `NULL` 使用 [IS NULL](operators/index.md#operator-is-null) 和 [IS NOT NULL](operators/index.md) 运算符及相关功能 `isNull` 和 `isNotNull`. +### NULL值 {#null-literal} -## 功能 {#functions} +代表不存在的值 -函数调用像一个标识符一样写入，并在圆括号中包含一个参数列表（可能是空的）。 与标准SQL相比，括号是必需的，即使是空的参数列表。 示例: `now()`. -有常规函数和聚合函数（请参阅部分 “Aggregate functions”). 某些聚合函数可以包含括号中的两个参数列表。 示例: `quantile (0.9) (x)`. 这些聚合函数被调用 “parametric” 函数，并在第一个列表中的参数被调用 “parameters”. 不带参数的聚合函数的语法与常规函数的语法相同。 +为了能在表字段中存储NULL值，该字段必须声明为 [空值](../sql-reference/data-types/nullable.md) 类型 +根据数据的格式（输入或输出），NULL值有不同的表现形式。更多信息参见文档 [数据格式](../interfaces/formats.md#formats) -## 运营商 {#operators} +在处理 `NULL`时存在很多细微差别。例如，比较运算的至少一个参数为 `NULL` ，该结果也是 `NULL` 。与之类似的还有乘法运算, 加法运算,以及其它运算。更多信息，请参阅每种运算的文档部分。 -在查询解析过程中，运算符会转换为相应的函数，同时考虑它们的优先级和关联性。 -例如，表达式 `1 + 2 * 3 + 4` 转化为 `plus(plus(1, multiply(2, 3)), 4)`. +在语句中，可以通过 [是否为NULL](operators/index.md#operator-is-null) 以及 [是否不为NULL](operators/index.md) 运算符，以及 `isNull` 、 `isNotNull` 函数来检查 `NULL` 值 -## 数据类型和数据库表引擎 {#data_types-and-database-table-engines} +## 函数 {#functions} +函数调用的写法，类似于变量并带有被圆括号包含的参数列表（可能为空）。与标准SQL不同，圆括号是必须的，不管参数列表是否为空。例如： `now()`。 -数据类型和表引擎 `CREATE` 查询的编写方式与标识符或函数相同。 换句话说，它们可能包含也可能不包含括号中的参数列表。 有关详细信息，请参阅部分 “Data types,” “Table engines,” 和 “CREATE”. +函数分为常规函数和聚合函数（参见“Aggregate functions”一章）。有些聚合函数包含2个参数列表，第一个参数列表中的参数被称为“parameters”。不包含“parameters”的聚合函数语法和常规函数是一样的。 + + +## 运算符 {#operators} + +在查询解析阶段，运算符会被转换成对应的函数，使用时请注意它们的优先级。例如： +表达式 `1 + 2 * 3 + 4` 会被解析成 `plus(plus(1, multiply(2, 3)), 4)`. + + +## 数据类型及数据库/表引擎 {#data_types-and-database-table-engines} + +`CREATE` 语句中的数据类型和表引擎写法与变量或函数类似。 +换句话说，它们可以用括号包含参数列表。更多信息，参见“数据类型,” “数据表引擎” 和 “CREATE语句”等章节 ## 表达式别名 {#syntax-expression_aliases} -别名是查询中表达式的用户定义名称。 +别名是用户对表达式的自定义名称 ``` sql expr AS alias ``` -- `AS` — The keyword for defining aliases. You can define the alias for a table name or a column name in a `SELECT` 子句不使用 `AS` 关键字。 +- `AS` — 用于定义别名的关键字。可以对表或select语句中的列定义别名(`AS` 可以省略） + 例如, `SELECT table_name_alias.column_name FROM table_name table_name_alias`. - For example, `SELECT table_name_alias.column_name FROM table_name table_name_alias`. + 在 [CAST函数](sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) 中，`AS`有其它含义。请参见该函数的说明部分。 - In the [CAST](sql_reference/functions/type_conversion_functions.md#type_conversion_function-cast) function, the `AS` keyword has another meaning. See the description of the function. -- `expr` — Any expression supported by ClickHouse. +- `expr` — 任意CH支持的表达式. - For example, `SELECT column_name * 2 AS double FROM some_table`. + 例如, `SELECT column_name * 2 AS double FROM some_table`. -- `alias` — Name for `expr`. 别名应符合 [标识符](#syntax-identifiers) 语法 +- `alias` — `expr` 的名称。别名必须符合 [变量名]](#syntax-identifiers) 语法. - For example, `SELECT "table t".column_name FROM table_name AS "table t"`. + 例如, `SELECT "table t".column_name FROM table_name AS "table t"`. -### 使用注意事项 {#notes-on-usage} +### 用法注意 {#notes-on-usage} -别名对于查询或子查询是全局的，您可以在查询的任何部分中为任何表达式定义别名。 例如, `SELECT (1 AS n) + 2, n`. +别名在当前查询或子查询中是全局可见的，你可以在查询语句的任何位置对表达式定义别名 -别名在子查询和子查询之间不可见。 例如，在执行查询时 `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` ClickHouse生成异常 `Unknown identifier: num`. +别名在当前查询的子查询及不同子查询中是不可见的。例如，执行如下查询SQL: `SELECT (SELECT sum(b.a) + num FROM b) - a.a AS num FROM a` ,CH会提示异常 `Unknown identifier: num`. -如果为结果列定义了别名 `SELECT` 子查询的子句，这些列在外部查询中可见。 例如, `SELECT n + m FROM (SELECT 1 AS n, 2 AS m)`. - -小心使用与列或表名相同的别名。 让我们考虑以下示例: +如果给select子查询语句的结果列定义其别名，那么在外层可以使用该别名。例如, `SELECT n + m FROM (SELECT 1 AS n, 2 AS m)`. +注意列的别名和表的别名相同时的情形，考虑如下示例： ``` sql CREATE TABLE t ( @@ -172,16 +178,18 @@ Received exception from server (version 18.14.17): Code: 184. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: Aggregate function sum(b) is found inside another aggregate function in query. ``` -在这个例子中，我们声明表 `t` 带柱 `b`. 然后，在选择数据时，我们定义了 `sum(b) AS b` 别名 由于别名是全局的，ClickHouse替换了文字 `b` 在表达式中 `argMax(a, b)` 用表达式 `sum(b)`. 这种替换导致异常。 +在这个示例中，先声明了表 `t` 以及列 `b`。然后，在查询数据时，又定义了别名 `sum(b) AS b`。由于别名是全局的，CH使用表达式 `sum(b)` 来替换表达式 `argMax(a, b)` 中的变量 `b`。这种替换导致出现异常。 ## 星号 {#asterisk} -在一个 `SELECT` 查询中，星号可以替换表达式。 有关详细信息，请参阅部分 “SELECT”. +select查询中，星号可以代替表达式使用。详情请参见“select”部分 + ## 表达式 {#syntax-expressions} -表达式是函数、标识符、文字、运算符的应用程序、括号中的表达式、子查询或星号。 它还可以包含别名。 -表达式列表是一个或多个用逗号分隔的表达式。 -函数和运算符，反过来，可以有表达式作为参数。 -[原始文章](https://clickhouse.tech/docs/en/sql_reference/syntax/) +An expression is a function, identifier, literal, application of an operator, expression in brackets, subquery, or asterisk. It can also contain an alias. +A list of expressions is one or more expressions separated by commas. +Functions and operators, in turn, can have expressions as arguments. + +[原始文档](https://clickhouse.tech/docs/en/sql_reference/syntax/) From 6af27d6c324222eec5c51284391518741d2a31b1 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 15 Jun 2020 18:22:11 +0300 Subject: [PATCH 221/329] Update simpleaggregatefunction.md --- docs/en/sql-reference/data-types/simpleaggregatefunction.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/sql-reference/data-types/simpleaggregatefunction.md b/docs/en/sql-reference/data-types/simpleaggregatefunction.md index 5f4c408f939..8b7e498e535 100644 --- a/docs/en/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/en/sql-reference/data-types/simpleaggregatefunction.md @@ -12,6 +12,8 @@ The following aggregate functions are supported: - [`groupBitAnd`](../../sql-reference/aggregate-functions/reference.md#groupbitand) - [`groupBitOr`](../../sql-reference/aggregate-functions/reference.md#groupbitor) - [`groupBitXor`](../../sql-reference/aggregate-functions/reference.md#groupbitxor) +- [`groupArrayArray`](../../sql-reference/aggregate-functions/reference.md#agg_function-grouparray) +- [`groupUniqArrayArray`](../../sql-reference/aggregate-functions/reference.md#groupuniqarrayx-groupuniqarraymax-sizex) Values of the `SimpleAggregateFunction(func, Type)` look and stored the same way as `Type`, so you do not need to apply functions with `-Merge`/`-State` suffixes. `SimpleAggregateFunction` has better performance than `AggregateFunction` with same aggregation function. From 6e18001bd81809e4b8d85c5b388e2111f685cb6f Mon Sep 17 00:00:00 2001 From: long2ice Date: Mon, 15 Jun 2020 23:25:13 +0800 Subject: [PATCH 222/329] add mysql2ch in docs (#11680) * add mysql2ch add mysql2ch * Update integrations.md * Update integrations.md * Update integrations.md * Update integrations.md * Update integrations.md * Update integrations.md * Update integrations.md --- docs/en/interfaces/third-party/integrations.md | 1 + docs/es/interfaces/third-party/integrations.md | 1 + docs/fa/interfaces/third-party/integrations.md | 1 + docs/fr/interfaces/third-party/integrations.md | 1 + docs/ja/interfaces/third-party/integrations.md | 1 + docs/ru/interfaces/third-party/integrations.md | 1 + docs/tr/interfaces/third-party/integrations.md | 1 + docs/zh/interfaces/third-party/integrations.md | 1 + 8 files changed, 8 insertions(+) diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index 716e774871b..17e7f1f18cc 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -12,6 +12,7 @@ toc_title: Integrations - Relational database management systems - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/es/interfaces/third-party/integrations.md b/docs/es/interfaces/third-party/integrations.md index 716e774871b..17e7f1f18cc 100644 --- a/docs/es/interfaces/third-party/integrations.md +++ b/docs/es/interfaces/third-party/integrations.md @@ -12,6 +12,7 @@ toc_title: Integrations - Relational database management systems - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/fa/interfaces/third-party/integrations.md b/docs/fa/interfaces/third-party/integrations.md index 657432c7958..df864ef71e6 100644 --- a/docs/fa/interfaces/third-party/integrations.md +++ b/docs/fa/interfaces/third-party/integrations.md @@ -14,6 +14,7 @@ toc_title: "\u06CC\u06A9\u067E\u0627\u0631\u0686\u06AF\u06CC" - سیستم های مدیریت پایگاه داده رابطه ای - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [در حال بارگذاری](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [تاتر-خروجی زیر-داده خوان](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/fr/interfaces/third-party/integrations.md b/docs/fr/interfaces/third-party/integrations.md index f252fd6229b..8332ffe5e59 100644 --- a/docs/fr/interfaces/third-party/integrations.md +++ b/docs/fr/interfaces/third-party/integrations.md @@ -14,6 +14,7 @@ toc_title: "Int\xE9gration" - Systèmes de gestion de bases de données relationnelles - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-lecteur de données](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-réplicateur](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/ja/interfaces/third-party/integrations.md b/docs/ja/interfaces/third-party/integrations.md index 3e38d578093..2ac2ad24410 100644 --- a/docs/ja/interfaces/third-party/integrations.md +++ b/docs/ja/interfaces/third-party/integrations.md @@ -14,6 +14,7 @@ toc_title: "\u7D71\u5408" - リレーショナルデータベース管理システム - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-データリーダー](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-レプリケーター](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md index 39449b54df8..19a72edc4d3 100644 --- a/docs/ru/interfaces/third-party/integrations.md +++ b/docs/ru/interfaces/third-party/integrations.md @@ -7,6 +7,7 @@ - Реляционные системы управления базами данных - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/tr/interfaces/third-party/integrations.md b/docs/tr/interfaces/third-party/integrations.md index 8a1d5c239f6..a5e5a60c72f 100644 --- a/docs/tr/interfaces/third-party/integrations.md +++ b/docs/tr/interfaces/third-party/integrations.md @@ -14,6 +14,7 @@ toc_title: Entegrasyonlar - İlişkisel veritabanı yönetim sistemleri - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-çoğaltıcı](https://github.com/larsnovikov/horgh-replicator) diff --git a/docs/zh/interfaces/third-party/integrations.md b/docs/zh/interfaces/third-party/integrations.md index 014fdc88304..e0f308fecde 100644 --- a/docs/zh/interfaces/third-party/integrations.md +++ b/docs/zh/interfaces/third-party/integrations.md @@ -7,6 +7,7 @@ - 关系数据库管理系统 - [MySQL](https://www.mysql.com) + - [mysql2ch](https://github.com/long2ice/mysql2ch) - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support) - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader) - [horgh-复制器](https://github.com/larsnovikov/horgh-replicator) From 3878254e0cf09213dab28ea26d28961fc0c42133 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 15 Jun 2020 19:21:52 +0300 Subject: [PATCH 223/329] Trigger CI --- .../0_stateless/01315_count_distinct_return_not_nullable.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql index 932cd2f69f9..0558d2cfd15 100644 --- a/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql +++ b/tests/queries/0_stateless/01315_count_distinct_return_not_nullable.sql @@ -7,13 +7,11 @@ SELECT uniqExact(number >= 5 ? number : NULL) FROM numbers(10); SELECT count(DISTINCT number >= 5 ? number : NULL) FROM numbers(10); SELECT '---'; - SELECT count(NULL); SELECT uniq(NULL); SELECT count(DISTINCT NULL); SELECT '---'; - SELECT avg(NULL); SELECT sum(NULL); SELECT corr(NULL, NULL); From 9f31184d7624e8220392108fd70942e87774958c Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 30 May 2020 17:18:08 +0300 Subject: [PATCH 224/329] Support for multiple names in one CREATE/ALTER command. --- src/Access/RowPolicy.cpp | 16 -- src/Access/RowPolicy.h | 18 ++ .../InterpreterCreateQuotaQuery.cpp | 55 ++++--- .../InterpreterCreateRoleQuery.cpp | 52 +++--- .../InterpreterCreateRowPolicyQuery.cpp | 58 +++---- .../InterpreterCreateSettingsProfileQuery.cpp | 65 ++++---- .../InterpreterCreateUserQuery.cpp | 86 +++++----- .../InterpreterDropAccessEntityQuery.cpp | 32 ++-- ...InterpreterShowCreateAccessEntityQuery.cpp | 59 ++++--- .../InterpreterShowCreateAccessEntityQuery.h | 2 +- src/Parsers/ASTCreateQuotaQuery.cpp | 16 +- src/Parsers/ASTCreateQuotaQuery.h | 2 +- src/Parsers/ASTCreateRoleQuery.cpp | 15 +- src/Parsers/ASTCreateRoleQuery.h | 2 +- src/Parsers/ASTCreateRowPolicyQuery.cpp | 15 +- src/Parsers/ASTCreateRowPolicyQuery.h | 7 +- src/Parsers/ASTCreateSettingsProfileQuery.cpp | 15 +- src/Parsers/ASTCreateSettingsProfileQuery.h | 2 +- src/Parsers/ASTCreateUserQuery.cpp | 4 +- src/Parsers/ASTCreateUserQuery.h | 3 +- src/Parsers/ASTDropAccessEntityQuery.cpp | 50 +++--- src/Parsers/ASTDropAccessEntityQuery.h | 5 +- src/Parsers/ASTRowPolicyName.cpp | 134 +++++++++++++++ src/Parsers/ASTRowPolicyName.h | 49 ++++++ .../ASTShowCreateAccessEntityQuery.cpp | 35 +++- src/Parsers/ASTShowCreateAccessEntityQuery.h | 12 +- src/Parsers/ASTUserNameWithHost.cpp | 74 +++++++++ src/Parsers/ASTUserNameWithHost.h | 53 ++++++ src/Parsers/ParserCreateQuotaQuery.cpp | 8 +- src/Parsers/ParserCreateRoleQuery.cpp | 8 +- src/Parsers/ParserCreateRowPolicyQuery.cpp | 20 ++- .../ParserCreateSettingsProfileQuery.cpp | 8 +- src/Parsers/ParserCreateUserQuery.cpp | 31 ++-- src/Parsers/ParserDropAccessEntityQuery.cpp | 83 ++-------- src/Parsers/ParserRowPolicyName.cpp | 154 ++++++++++++++++++ src/Parsers/ParserRowPolicyName.h | 47 ++++++ .../ParserShowCreateAccessEntityQuery.cpp | 29 ++-- src/Parsers/ParserShowGrantsQuery.cpp | 4 +- src/Parsers/ParserUserNameWithHost.cpp | 56 +++++++ src/Parsers/ParserUserNameWithHost.h | 26 +++ .../parseIdentifierOrStringLiteral.cpp | 22 +++ src/Parsers/parseIdentifierOrStringLiteral.h | 3 + src/Parsers/parseUserName.cpp | 56 +++---- src/Parsers/parseUserName.h | 17 +- src/Parsers/ya.make | 4 + .../test_access_control_on_cluster/test.py | 1 - 46 files changed, 1090 insertions(+), 423 deletions(-) create mode 100644 src/Parsers/ASTRowPolicyName.cpp create mode 100644 src/Parsers/ASTRowPolicyName.h create mode 100644 src/Parsers/ASTUserNameWithHost.cpp create mode 100644 src/Parsers/ASTUserNameWithHost.h create mode 100644 src/Parsers/ParserRowPolicyName.cpp create mode 100644 src/Parsers/ParserRowPolicyName.h create mode 100644 src/Parsers/ParserUserNameWithHost.cpp create mode 100644 src/Parsers/ParserUserNameWithHost.h diff --git a/src/Access/RowPolicy.cpp b/src/Access/RowPolicy.cpp index acacaf01c6c..7441f915a46 100644 --- a/src/Access/RowPolicy.cpp +++ b/src/Access/RowPolicy.cpp @@ -11,22 +11,6 @@ namespace ErrorCodes } -String RowPolicy::NameParts::getName() const -{ - String name; - name.reserve(database.length() + table_name.length() + short_name.length() + 6); - name += backQuoteIfNeed(short_name); - name += " ON "; - if (!database.empty()) - { - name += backQuoteIfNeed(database); - name += '.'; - } - name += backQuoteIfNeed(table_name); - return name; -} - - void RowPolicy::setDatabase(const String & database) { name_parts.database = database; diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h index 7febf5991fb..9d5b00b427d 100644 --- a/src/Access/RowPolicy.h +++ b/src/Access/RowPolicy.h @@ -23,7 +23,9 @@ struct RowPolicy : public IAccessEntity String database; String table_name; + bool empty() const { return short_name.empty(); } String getName() const; + String toString() const { return getName(); } auto toTuple() const { return std::tie(short_name, database, table_name); } friend bool operator ==(const NameParts & left, const NameParts & right) { return left.toTuple() == right.toTuple(); } friend bool operator !=(const NameParts & left, const NameParts & right) { return left.toTuple() != right.toTuple(); } @@ -153,4 +155,20 @@ inline String toString(RowPolicy::ConditionType type) return RowPolicy::ConditionTypeInfo::get(type).raw_name; } + +inline String RowPolicy::NameParts::getName() const +{ + String name; + name.reserve(database.length() + table_name.length() + short_name.length() + 6); + name += backQuoteIfNeed(short_name); + name += " ON "; + if (!database.empty()) + { + name += backQuoteIfNeed(database); + name += '.'; + } + name += backQuoteIfNeed(table_name); + return name; +} + } diff --git a/src/Interpreters/InterpreterCreateQuotaQuery.cpp b/src/Interpreters/InterpreterCreateQuotaQuery.cpp index 907532c3d89..0cca163beec 100644 --- a/src/Interpreters/InterpreterCreateQuotaQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuotaQuery.cpp @@ -15,15 +15,18 @@ namespace DB { namespace { -void updateQuotaFromQueryImpl(Quota & quota, const ASTCreateQuotaQuery & query, const std::optional & roles_from_query = {}) + void updateQuotaFromQueryImpl( + Quota & quota, + const ASTCreateQuotaQuery & query, + const String & override_name, + const std::optional & override_to_roles) { - if (query.alter) - { - if (!query.new_name.empty()) - quota.setName(query.new_name); - } - else - quota.setName(query.name); + if (!override_name.empty()) + quota.setName(override_name); + else if (!query.new_name.empty()) + quota.setName(query.new_name); + else if (query.names.size() == 1) + quota.setName(query.names.front()); if (query.key_type) quota.key_type = *query.key_type; @@ -59,15 +62,10 @@ void updateQuotaFromQueryImpl(Quota & quota, const ASTCreateQuotaQuery & query, quota_limits.max[resource_type] = query_limits.max[resource_type]; } - const ExtendedRoleSet * roles = nullptr; - std::optional temp_role_set; - if (roles_from_query) - roles = &*roles_from_query; + if (override_to_roles) + quota.to_roles = *override_to_roles; else if (query.roles) - roles = &temp_role_set.emplace(*query.roles); - - if (roles) - quota.to_roles = *roles; + quota.to_roles = *query.roles; } } @@ -93,28 +91,33 @@ BlockIO InterpreterCreateQuotaQuery::execute() auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto updated_quota = typeid_cast>(entity->clone()); - updateQuotaFromQueryImpl(*updated_quota, query, roles_from_query); + updateQuotaFromQueryImpl(*updated_quota, query, {}, roles_from_query); return updated_quota; }; if (query.if_exists) { - if (auto id = access_control.find(query.name)) - access_control.tryUpdate(*id, update_func); + auto ids = access_control.find(query.names); + access_control.tryUpdate(ids, update_func); } else - access_control.update(access_control.getID(query.name), update_func); + access_control.update(access_control.getIDs(query.names), update_func); } else { - auto new_quota = std::make_shared(); - updateQuotaFromQueryImpl(*new_quota, query, roles_from_query); + std::vector new_quotas; + for (const String & name : query.names) + { + auto new_quota = std::make_shared(); + updateQuotaFromQueryImpl(*new_quota, query, name, roles_from_query); + new_quotas.emplace_back(std::move(new_quota)); + } if (query.if_not_exists) - access_control.tryInsert(new_quota); + access_control.tryInsert(new_quotas); else if (query.or_replace) - access_control.insertOrReplace(new_quota); + access_control.insertOrReplace(new_quotas); else - access_control.insert(new_quota); + access_control.insert(new_quotas); } return {}; @@ -123,7 +126,7 @@ BlockIO InterpreterCreateQuotaQuery::execute() void InterpreterCreateQuotaQuery::updateQuotaFromQuery(Quota & quota, const ASTCreateQuotaQuery & query) { - updateQuotaFromQueryImpl(quota, query); + updateQuotaFromQueryImpl(quota, query, {}, {}); } } diff --git a/src/Interpreters/InterpreterCreateRoleQuery.cpp b/src/Interpreters/InterpreterCreateRoleQuery.cpp index ed9135b2bb6..2fa04eebae1 100644 --- a/src/Interpreters/InterpreterCreateRoleQuery.cpp +++ b/src/Interpreters/InterpreterCreateRoleQuery.cpp @@ -13,25 +13,20 @@ namespace void updateRoleFromQueryImpl( Role & role, const ASTCreateRoleQuery & query, - const std::optional & settings_from_query = {}) + const String & override_name, + const std::optional & override_settings) { - if (query.alter) - { - if (!query.new_name.empty()) - role.setName(query.new_name); - } - else - role.setName(query.name); + if (!override_name.empty()) + role.setName(override_name); + else if (!query.new_name.empty()) + role.setName(query.new_name); + else if (query.names.size() == 1) + role.setName(query.names.front()); - const SettingsProfileElements * settings = nullptr; - std::optional temp_settings; - if (settings_from_query) - settings = &*settings_from_query; + if (override_settings) + role.settings = *override_settings; else if (query.settings) - settings = &temp_settings.emplace(*query.settings); - - if (settings) - role.settings = *settings; + role.settings = *query.settings; } } @@ -57,28 +52,33 @@ BlockIO InterpreterCreateRoleQuery::execute() auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto updated_role = typeid_cast>(entity->clone()); - updateRoleFromQueryImpl(*updated_role, query, settings_from_query); + updateRoleFromQueryImpl(*updated_role, query, {}, settings_from_query); return updated_role; }; if (query.if_exists) { - if (auto id = access_control.find(query.name)) - access_control.tryUpdate(*id, update_func); + auto ids = access_control.find(query.names); + access_control.tryUpdate(ids, update_func); } else - access_control.update(access_control.getID(query.name), update_func); + access_control.update(access_control.getIDs(query.names), update_func); } else { - auto new_role = std::make_shared(); - updateRoleFromQueryImpl(*new_role, query, settings_from_query); + std::vector new_roles; + for (const auto & name : query.names) + { + auto new_role = std::make_shared(); + updateRoleFromQueryImpl(*new_role, query, name, settings_from_query); + new_roles.emplace_back(std::move(new_role)); + } if (query.if_not_exists) - access_control.tryInsert(new_role); + access_control.tryInsert(new_roles); else if (query.or_replace) - access_control.insertOrReplace(new_role); + access_control.insertOrReplace(new_roles); else - access_control.insert(new_role); + access_control.insert(new_roles); } return {}; @@ -87,6 +87,6 @@ BlockIO InterpreterCreateRoleQuery::execute() void InterpreterCreateRoleQuery::updateRoleFromQuery(Role & role, const ASTCreateRoleQuery & query) { - updateRoleFromQueryImpl(role, query); + updateRoleFromQueryImpl(role, query, {}, {}); } } diff --git a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp index 778a32019d9..bfd7d60b397 100644 --- a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp +++ b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -16,15 +17,15 @@ namespace void updateRowPolicyFromQueryImpl( RowPolicy & policy, const ASTCreateRowPolicyQuery & query, - const std::optional & roles_from_query = {}) + const RowPolicy::NameParts & override_name, + const std::optional & override_to_roles) { - if (query.alter) - { - if (!query.new_short_name.empty()) - policy.setShortName(query.new_short_name); - } - else - policy.setNameParts(query.name_parts); + if (!override_name.empty()) + policy.setNameParts(override_name); + else if (!query.new_short_name.empty()) + policy.setShortName(query.new_short_name); + else if (query.names->name_parts.size() == 1) + policy.setNameParts(query.names->name_parts.front()); if (query.is_restrictive) policy.setRestrictive(*query.is_restrictive); @@ -36,15 +37,10 @@ namespace policy.conditions[condition_type] = *condition ? serializeAST(**condition) : String{}; } - const ExtendedRoleSet * roles = nullptr; - std::optional temp_role_set; - if (roles_from_query) - roles = &*roles_from_query; + if (override_to_roles) + policy.to_roles = *override_to_roles; else if (query.roles) - roles = &temp_role_set.emplace(*query.roles); - - if (roles) - policy.to_roles = *roles; + policy.to_roles = *query.roles; } } @@ -61,40 +57,46 @@ BlockIO InterpreterCreateRowPolicyQuery::execute() return executeDDLQueryOnCluster(query_ptr, context); } + assert(query.names->cluster.empty()); std::optional roles_from_query; if (query.roles) roles_from_query = ExtendedRoleSet{*query.roles, access_control, context.getUserID()}; - if (query.name_parts.database.empty()) - query.name_parts.database = context.getCurrentDatabase(); + query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); if (query.alter) { auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto updated_policy = typeid_cast>(entity->clone()); - updateRowPolicyFromQueryImpl(*updated_policy, query, roles_from_query); + updateRowPolicyFromQueryImpl(*updated_policy, query, {}, roles_from_query); return updated_policy; }; + Strings names = query.names->toStrings(); if (query.if_exists) { - if (auto id = access_control.find(query.name_parts.getName())) - access_control.tryUpdate(*id, update_func); + auto ids = access_control.find(names); + access_control.tryUpdate(ids, update_func); } else - access_control.update(access_control.getID(query.name_parts.getName()), update_func); + access_control.update(access_control.getIDs(names), update_func); } else { - auto new_policy = std::make_shared(); - updateRowPolicyFromQueryImpl(*new_policy, query, roles_from_query); + std::vector new_policies; + for (const auto & name_parts : query.names->name_parts) + { + auto new_policy = std::make_shared(); + updateRowPolicyFromQueryImpl(*new_policy, query, name_parts, roles_from_query); + new_policies.emplace_back(std::move(new_policy)); + } if (query.if_not_exists) - access_control.tryInsert(new_policy); + access_control.tryInsert(new_policies); else if (query.or_replace) - access_control.insertOrReplace(new_policy); + access_control.insertOrReplace(new_policies); else - access_control.insert(new_policy); + access_control.insert(new_policies); } return {}; @@ -103,7 +105,7 @@ BlockIO InterpreterCreateRowPolicyQuery::execute() void InterpreterCreateRowPolicyQuery::updateRowPolicyFromQuery(RowPolicy & policy, const ASTCreateRowPolicyQuery & query) { - updateRowPolicyFromQueryImpl(policy, query); + updateRowPolicyFromQueryImpl(policy, query, {}, {}); } } diff --git a/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp b/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp index cb0b5587bdc..ac2a4249986 100644 --- a/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp +++ b/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp @@ -15,36 +15,26 @@ namespace void updateSettingsProfileFromQueryImpl( SettingsProfile & profile, const ASTCreateSettingsProfileQuery & query, - const std::optional & settings_from_query = {}, - const std::optional & roles_from_query = {}) + const String & override_name, + const std::optional & override_settings, + const std::optional & override_to_roles) { - if (query.alter) - { - if (!query.new_name.empty()) - profile.setName(query.new_name); - } - else - profile.setName(query.name); + if (!override_name.empty()) + profile.setName(override_name); + else if (!query.new_name.empty()) + profile.setName(query.new_name); + else if (query.names.size() == 1) + profile.setName(query.names.front()); - const SettingsProfileElements * settings = nullptr; - std::optional temp_settings; - if (settings_from_query) - settings = &*settings_from_query; + if (override_settings) + profile.elements = *override_settings; else if (query.settings) - settings = &temp_settings.emplace(*query.settings); + profile.elements = *query.settings; - if (settings) - profile.elements = *settings; - - const ExtendedRoleSet * roles = nullptr; - std::optional temp_role_set; - if (roles_from_query) - roles = &*roles_from_query; + if (override_to_roles) + profile.to_roles = *override_to_roles; else if (query.to_roles) - roles = &temp_role_set.emplace(*query.to_roles); - - if (roles) - profile.to_roles = *roles; + profile.to_roles = *query.to_roles; } } @@ -77,28 +67,33 @@ BlockIO InterpreterCreateSettingsProfileQuery::execute() auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto updated_profile = typeid_cast>(entity->clone()); - updateSettingsProfileFromQueryImpl(*updated_profile, query, settings_from_query, roles_from_query); + updateSettingsProfileFromQueryImpl(*updated_profile, query, {}, settings_from_query, roles_from_query); return updated_profile; }; if (query.if_exists) { - if (auto id = access_control.find(query.name)) - access_control.tryUpdate(*id, update_func); + auto ids = access_control.find(query.names); + access_control.tryUpdate(ids, update_func); } else - access_control.update(access_control.getID(query.name), update_func); + access_control.update(access_control.getIDs(query.names), update_func); } else { - auto new_profile = std::make_shared(); - updateSettingsProfileFromQueryImpl(*new_profile, query, settings_from_query, roles_from_query); + std::vector new_profiles; + for (const auto & name : query.names) + { + auto new_profile = std::make_shared(); + updateSettingsProfileFromQueryImpl(*new_profile, query, name, settings_from_query, roles_from_query); + new_profiles.emplace_back(std::move(new_profile)); + } if (query.if_not_exists) - access_control.tryInsert(new_profile); + access_control.tryInsert(new_profiles); else if (query.or_replace) - access_control.insertOrReplace(new_profile); + access_control.insertOrReplace(new_profiles); else - access_control.insert(new_profile); + access_control.insert(new_profiles); } return {}; @@ -107,6 +102,6 @@ BlockIO InterpreterCreateSettingsProfileQuery::execute() void InterpreterCreateSettingsProfileQuery::updateSettingsProfileFromQuery(SettingsProfile & SettingsProfile, const ASTCreateSettingsProfileQuery & query) { - updateSettingsProfileFromQueryImpl(SettingsProfile, query); + updateSettingsProfileFromQueryImpl(SettingsProfile, query, {}, {}, {}); } } diff --git a/src/Interpreters/InterpreterCreateUserQuery.cpp b/src/Interpreters/InterpreterCreateUserQuery.cpp index 7c488ddf8e9..8b57703f08c 100644 --- a/src/Interpreters/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/InterpreterCreateUserQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -17,51 +18,50 @@ namespace void updateUserFromQueryImpl( User & user, const ASTCreateUserQuery & query, - const std::optional & default_roles_from_query = {}, - const std::optional & settings_from_query = {}) + const std::shared_ptr & override_name, + const std::optional & override_default_roles, + const std::optional & override_settings) { - if (query.alter) - { - if (!query.new_name.empty()) - user.setName(query.new_name); - } - else - user.setName(query.name); + if (override_name) + user.setName(override_name->toString()); + else if (!query.new_name.empty()) + user.setName(query.new_name); + else if (query.names->size() == 1) + user.setName(query.names->front()->toString()); if (query.authentication) user.authentication = *query.authentication; - if (query.hosts) + if (override_name && !override_name->host_pattern.empty()) + { + user.allowed_client_hosts = AllowedClientHosts{}; + user.allowed_client_hosts.addLikePattern(override_name->host_pattern); + } + else if (query.hosts) user.allowed_client_hosts = *query.hosts; + if (query.remove_hosts) user.allowed_client_hosts.remove(*query.remove_hosts); if (query.add_hosts) user.allowed_client_hosts.add(*query.add_hosts); - const ExtendedRoleSet * default_roles = nullptr; - std::optional temp_role_set; - if (default_roles_from_query) - default_roles = &*default_roles_from_query; - else if (query.default_roles) - default_roles = &temp_role_set.emplace(*query.default_roles); - - if (default_roles) + auto set_default_roles = [&](const ExtendedRoleSet & default_roles_) { - if (!query.alter && !default_roles->all) - user.granted_roles.grant(default_roles->getMatchingIDs()); + if (!query.alter && !default_roles_.all) + user.granted_roles.grant(default_roles_.getMatchingIDs()); - InterpreterSetRoleQuery::updateUserSetDefaultRoles(user, *default_roles); - } + InterpreterSetRoleQuery::updateUserSetDefaultRoles(user, default_roles_); + }; - const SettingsProfileElements * settings = nullptr; - std::optional temp_settings; - if (settings_from_query) - settings = &*settings_from_query; + if (override_default_roles) + set_default_roles(*override_default_roles); + else if (query.default_roles) + set_default_roles(*query.default_roles); + + if (override_settings) + user.settings = *override_settings; else if (query.settings) - settings = &temp_settings.emplace(*query.settings); - - if (settings) - user.settings = *settings; + user.settings = *query.settings; } } @@ -96,28 +96,34 @@ BlockIO InterpreterCreateUserQuery::execute() auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto updated_user = typeid_cast>(entity->clone()); - updateUserFromQueryImpl(*updated_user, query, default_roles_from_query, settings_from_query); + updateUserFromQueryImpl(*updated_user, query, {}, default_roles_from_query, settings_from_query); return updated_user; }; + Strings names = query.names->toStrings(); if (query.if_exists) { - if (auto id = access_control.find(query.name)) - access_control.tryUpdate(*id, update_func); + auto ids = access_control.find(names); + access_control.tryUpdate(ids, update_func); } else - access_control.update(access_control.getID(query.name), update_func); + access_control.update(access_control.getIDs(names), update_func); } else { - auto new_user = std::make_shared(); - updateUserFromQueryImpl(*new_user, query, default_roles_from_query, settings_from_query); + std::vector new_users; + for (const auto & name : *query.names) + { + auto new_user = std::make_shared(); + updateUserFromQueryImpl(*new_user, query, name, default_roles_from_query, settings_from_query); + new_users.emplace_back(std::move(new_user)); + } if (query.if_not_exists) - access_control.tryInsert(new_user); + access_control.tryInsert(new_users); else if (query.or_replace) - access_control.insertOrReplace(new_user); + access_control.insertOrReplace(new_users); else - access_control.insert(new_user); + access_control.insert(new_users); } return {}; @@ -126,7 +132,7 @@ BlockIO InterpreterCreateUserQuery::execute() void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreateUserQuery & query) { - updateUserFromQueryImpl(user, query); + updateUserFromQueryImpl(user, query, {}, {}, {}); } } diff --git a/src/Interpreters/InterpreterDropAccessEntityQuery.cpp b/src/Interpreters/InterpreterDropAccessEntityQuery.cpp index be82147a322..d79d239ee12 100644 --- a/src/Interpreters/InterpreterDropAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterDropAccessEntityQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -30,26 +31,21 @@ BlockIO InterpreterDropAccessEntityQuery::execute() if (!query.cluster.empty()) return executeDDLQueryOnCluster(query_ptr, context); - if (query.type == EntityType::ROW_POLICY) - { - Strings names; - for (auto & name_parts : query.row_policies_name_parts) - { - if (name_parts.database.empty()) - name_parts.database = context.getCurrentDatabase(); - names.emplace_back(name_parts.getName()); - } - if (query.if_exists) - access_control.tryRemove(access_control.find(names)); - else - access_control.remove(access_control.getIDs(names)); - return {}; - } + query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); - if (query.if_exists) - access_control.tryRemove(access_control.find(query.type, query.names)); + auto do_drop = [&](const Strings & names) + { + if (query.if_exists) + access_control.tryRemove(access_control.find(query.type, names)); + else + access_control.remove(access_control.getIDs(query.type, names)); + }; + + if (query.type == EntityType::ROW_POLICY) + do_drop(query.row_policy_names->toStrings()); else - access_control.remove(access_control.getIDs(query.type, query.names)); + do_drop(query.names); + return {}; } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index e37c31aab22..d3ab5ac5001 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -6,8 +6,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -42,7 +44,8 @@ namespace bool attach_mode) { auto query = std::make_shared(); - query->name = user.getName(); + query->names = std::make_shared(); + query->names->push_back(user.getName()); query->attach = attach_mode; if (user.allowed_client_hosts != AllowedClientHosts::AnyHostTag{}) @@ -77,7 +80,7 @@ namespace ASTPtr getCreateQueryImpl(const Role & role, const AccessControlManager * manager, bool attach_mode) { auto query = std::make_shared(); - query->name = role.getName(); + query->names.emplace_back(role.getName()); query->attach = attach_mode; if (!role.settings.empty()) @@ -95,7 +98,7 @@ namespace ASTPtr getCreateQueryImpl(const SettingsProfile & profile, const AccessControlManager * manager, bool attach_mode) { auto query = std::make_shared(); - query->name = profile.getName(); + query->names.emplace_back(profile.getName()); query->attach = attach_mode; if (!profile.elements.empty()) @@ -126,7 +129,7 @@ namespace bool attach_mode) { auto query = std::make_shared(); - query->name = quota.getName(); + query->names.emplace_back(quota.getName()); query->attach = attach_mode; query->key_type = quota.key_type; @@ -160,7 +163,8 @@ namespace bool attach_mode) { auto query = std::make_shared(); - query->name_parts = policy.getNameParts(); + query->names = std::make_shared(); + query->names->name_parts.emplace_back(policy.getNameParts()); query->attach = attach_mode; if (policy.isRestrictive()) @@ -228,17 +232,17 @@ BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() { auto & show_query = query_ptr->as(); - /// Build a create query. - ASTPtr create_query = getCreateQuery(show_query); + /// Build a create queries. + ASTs create_queries = getCreateQueries(show_query); /// Build the result column. MutableColumnPtr column = ColumnString::create(); - if (create_query) + std::stringstream create_query_ss; + for (const auto & create_query : create_queries) { - std::stringstream create_query_ss; formatAST(*create_query, create_query_ss, false, true); - String create_query_str = create_query_ss.str(); - column->insert(create_query_str); + column->insert(create_query_ss.str()); + create_query_ss.str(""); } /// Prepare description of the result column. @@ -253,38 +257,49 @@ BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() } -ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(ASTShowCreateAccessEntityQuery & show_query) const +ASTs InterpreterShowCreateAccessEntityQuery::getCreateQueries(ASTShowCreateAccessEntityQuery & show_query) const { const auto & access_control = context.getAccessControlManager(); context.checkAccess(getRequiredAccess()); + show_query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); if (show_query.current_user) { auto user = context.getUser(); if (!user) - return nullptr; - return getCreateQueryImpl(*user, &access_control, false); + return {}; + return {getCreateQueryImpl(*user, &access_control, false)}; } if (show_query.current_quota) { auto usage = context.getQuotaUsage(); if (!usage) - return nullptr; + return {}; auto quota = access_control.read(usage->quota_id); - return getCreateQueryImpl(*quota, &access_control, false); + return {getCreateQueryImpl(*quota, &access_control, false)}; } + ASTs list; + if (show_query.type == EntityType::ROW_POLICY) { - if (show_query.row_policy_name_parts.database.empty()) - show_query.row_policy_name_parts.database = context.getCurrentDatabase(); - RowPolicyPtr policy = access_control.read(show_query.row_policy_name_parts.getName()); - return getCreateQueryImpl(*policy, &access_control, false); + for (const String & name : show_query.row_policy_names->toStrings()) + { + RowPolicyPtr policy = access_control.read(name); + list.push_back(getCreateQueryImpl(*policy, &access_control, false)); + } + } + else + { + for (const String & name : show_query.names) + { + auto entity = access_control.read(access_control.getID(show_query.type, name)); + list.push_back(getCreateQueryImpl(*entity, &access_control, false)); + } } - auto entity = access_control.read(access_control.getID(show_query.type, show_query.name)); - return getCreateQueryImpl(*entity, &access_control, false); + return list; } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h index ee28bfbdc4a..0d2978cff6c 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h @@ -29,7 +29,7 @@ public: private: BlockInputStreamPtr executeImpl(); - ASTPtr getCreateQuery(ASTShowCreateAccessEntityQuery & show_query) const; + ASTs getCreateQueries(ASTShowCreateAccessEntityQuery & show_query) const; AccessRightsElements getRequiredAccess() const; ASTPtr query_ptr; diff --git a/src/Parsers/ASTCreateQuotaQuery.cpp b/src/Parsers/ASTCreateQuotaQuery.cpp index bdfd1b32e96..d33af6126f1 100644 --- a/src/Parsers/ASTCreateQuotaQuery.cpp +++ b/src/Parsers/ASTCreateQuotaQuery.cpp @@ -23,6 +23,19 @@ namespace } + void formatNames(const Strings & names, const IAST::FormatSettings & settings) + { + settings.ostr << " "; + bool need_comma = false; + for (const String & name : names) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << backQuoteIfNeed(name); + } + } + + void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") @@ -130,8 +143,7 @@ void ASTCreateQuotaQuery::formatImpl(const FormatSettings & settings, FormatStat else if (or_replace) settings.ostr << (settings.hilite ? hilite_keyword : "") << " OR REPLACE" << (settings.hilite ? hilite_none : ""); - settings.ostr << " " << backQuoteIfNeed(name); - + formatNames(names, settings); formatOnCluster(settings); if (!new_name.empty()) diff --git a/src/Parsers/ASTCreateQuotaQuery.h b/src/Parsers/ASTCreateQuotaQuery.h index b9994e8ec3a..370083f4e25 100644 --- a/src/Parsers/ASTCreateQuotaQuery.h +++ b/src/Parsers/ASTCreateQuotaQuery.h @@ -38,7 +38,7 @@ public: using KeyType = Quota::KeyType; using ResourceAmount = Quota::ResourceAmount; - String name; + Strings names; String new_name; std::optional key_type; diff --git a/src/Parsers/ASTCreateRoleQuery.cpp b/src/Parsers/ASTCreateRoleQuery.cpp index f3873f7a3eb..5ccfd9c6bd5 100644 --- a/src/Parsers/ASTCreateRoleQuery.cpp +++ b/src/Parsers/ASTCreateRoleQuery.cpp @@ -7,6 +7,18 @@ namespace DB { namespace { + void formatNames(const Strings & names, const IAST::FormatSettings & settings) + { + settings.ostr << " "; + bool need_comma = false; + for (const String & name : names) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << backQuoteIfNeed(name); + } + } + void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") @@ -52,8 +64,7 @@ void ASTCreateRoleQuery::formatImpl(const FormatSettings & format, FormatState & else if (or_replace) format.ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : ""); - format.ostr << " " << backQuoteIfNeed(name); - + formatNames(names, format); formatOnCluster(format); if (!new_name.empty()) diff --git a/src/Parsers/ASTCreateRoleQuery.h b/src/Parsers/ASTCreateRoleQuery.h index ab306dd5dec..422bb0e681d 100644 --- a/src/Parsers/ASTCreateRoleQuery.h +++ b/src/Parsers/ASTCreateRoleQuery.h @@ -26,7 +26,7 @@ public: bool if_not_exists = false; bool or_replace = false; - String name; + Strings names; String new_name; std::shared_ptr settings; diff --git a/src/Parsers/ASTCreateRowPolicyQuery.cpp b/src/Parsers/ASTCreateRowPolicyQuery.cpp index 973f3c6b930..caa52e3ac58 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/ASTCreateRowPolicyQuery.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -154,13 +155,11 @@ void ASTCreateRowPolicyQuery::formatImpl(const FormatSettings & settings, Format else if (or_replace) settings.ostr << (settings.hilite ? hilite_keyword : "") << " OR REPLACE" << (settings.hilite ? hilite_none : ""); - const String & database = name_parts.database; - const String & table_name = name_parts.table_name; - const String & short_name = name_parts.short_name; - settings.ostr << " " << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " - << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") << table_name; + settings.ostr << " "; + names->format(settings); formatOnCluster(settings); + assert(names->cluster.empty()); if (!new_short_name.empty()) formatRenameTo(new_short_name, settings); @@ -180,4 +179,10 @@ void ASTCreateRowPolicyQuery::replaceCurrentUserTagWithName(const String & curre if (roles) roles->replaceCurrentUserTagWithName(current_user_name); } + +void ASTCreateRowPolicyQuery::replaceEmptyDatabaseWithCurrent(const String & current_database) const +{ + if (names) + names->replaceEmptyDatabaseWithCurrent(current_database); +} } diff --git a/src/Parsers/ASTCreateRowPolicyQuery.h b/src/Parsers/ASTCreateRowPolicyQuery.h index 8aa44b784aa..af561b47e12 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.h +++ b/src/Parsers/ASTCreateRowPolicyQuery.h @@ -9,6 +9,7 @@ namespace DB { +class ASTRowPolicyNames; class ASTExtendedRoleSet; /** CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] name ON [database.]table @@ -36,7 +37,7 @@ public: bool if_not_exists = false; bool or_replace = false; - RowPolicy::NameParts name_parts; + std::shared_ptr names; String new_short_name; std::optional is_restrictive; @@ -47,7 +48,9 @@ public: String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; - void replaceCurrentUserTagWithName(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } + + void replaceCurrentUserTagWithName(const String & current_user_name) const; + void replaceEmptyDatabaseWithCurrent(const String & current_database) const; }; } diff --git a/src/Parsers/ASTCreateSettingsProfileQuery.cpp b/src/Parsers/ASTCreateSettingsProfileQuery.cpp index 601425d3446..21d8c20ffc1 100644 --- a/src/Parsers/ASTCreateSettingsProfileQuery.cpp +++ b/src/Parsers/ASTCreateSettingsProfileQuery.cpp @@ -8,6 +8,18 @@ namespace DB { namespace { + void formatNames(const Strings & names, const IAST::FormatSettings & settings) + { + settings.ostr << " "; + bool need_comma = false; + for (const String & name : names) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << backQuoteIfNeed(name); + } + } + void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") @@ -59,8 +71,7 @@ void ASTCreateSettingsProfileQuery::formatImpl(const FormatSettings & format, Fo else if (or_replace) format.ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : ""); - format.ostr << " " << backQuoteIfNeed(name); - + formatNames(names, format); formatOnCluster(format); if (!new_name.empty()) diff --git a/src/Parsers/ASTCreateSettingsProfileQuery.h b/src/Parsers/ASTCreateSettingsProfileQuery.h index cd470283410..bb2a9474504 100644 --- a/src/Parsers/ASTCreateSettingsProfileQuery.h +++ b/src/Parsers/ASTCreateSettingsProfileQuery.h @@ -29,7 +29,7 @@ public: bool if_not_exists = false; bool or_replace = false; - String name; + Strings names; String new_name; std::shared_ptr settings; diff --git a/src/Parsers/ASTCreateUserQuery.cpp b/src/Parsers/ASTCreateUserQuery.cpp index e5c1178285b..60f61fbf51f 100644 --- a/src/Parsers/ASTCreateUserQuery.cpp +++ b/src/Parsers/ASTCreateUserQuery.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -212,7 +213,8 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & else if (or_replace) format.ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : ""); - format.ostr << " " << backQuoteIfNeed(name); + format.ostr << " "; + names->format(format); formatOnCluster(format); diff --git a/src/Parsers/ASTCreateUserQuery.h b/src/Parsers/ASTCreateUserQuery.h index 28ef6c059da..565c82bc98e 100644 --- a/src/Parsers/ASTCreateUserQuery.h +++ b/src/Parsers/ASTCreateUserQuery.h @@ -8,6 +8,7 @@ namespace DB { +class ASTUserNamesWithHost; class ASTExtendedRoleSet; class ASTSettingsProfileElements; @@ -34,7 +35,7 @@ public: bool if_not_exists = false; bool or_replace = false; - String name; + std::shared_ptr names; String new_name; std::optional authentication; diff --git a/src/Parsers/ASTDropAccessEntityQuery.cpp b/src/Parsers/ASTDropAccessEntityQuery.cpp index 9f7a1d86221..fe98d8b4158 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.cpp +++ b/src/Parsers/ASTDropAccessEntityQuery.cpp @@ -1,10 +1,25 @@ #include +#include #include namespace DB { -using EntityTypeInfo = IAccessEntity::TypeInfo; +namespace +{ + using EntityTypeInfo = IAccessEntity::TypeInfo; + + void formatNames(const Strings & names, const IAST::FormatSettings & settings) + { + bool need_comma = false; + for (const auto & name : names) + { + if (std::exchange(need_comma, true)) + settings.ostr << ','; + settings.ostr << ' ' << backQuoteIfNeed(name); + } + } +} String ASTDropAccessEntityQuery::getID(char) const @@ -28,32 +43,19 @@ void ASTDropAccessEntityQuery::formatImpl(const FormatSettings & settings, Forma if (type == EntityType::ROW_POLICY) { - bool need_comma = false; - for (const auto & name_parts : row_policies_name_parts) - { - if (need_comma) - settings.ostr << ','; - need_comma = true; - const String & database = name_parts.database; - const String & table_name = name_parts.table_name; - const String & short_name = name_parts.short_name; - settings.ostr << ' ' << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " - << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") - << backQuoteIfNeed(table_name); - } + settings.ostr << " "; + row_policy_names->format(settings); } else - { - bool need_comma = false; - for (const auto & name : names) - { - if (need_comma) - settings.ostr << ','; - need_comma = true; - settings.ostr << ' ' << backQuoteIfNeed(name); - } - } + formatNames(names, settings); formatOnCluster(settings); } + + +void ASTDropAccessEntityQuery::replaceEmptyDatabaseWithCurrent(const String & current_database) const +{ + if (row_policy_names) + row_policy_names->replaceEmptyDatabaseWithCurrent(current_database); +} } diff --git a/src/Parsers/ASTDropAccessEntityQuery.h b/src/Parsers/ASTDropAccessEntityQuery.h index 160b0c2e212..76a5f450566 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.h +++ b/src/Parsers/ASTDropAccessEntityQuery.h @@ -7,6 +7,7 @@ namespace DB { +class ASTRowPolicyNames; /** DROP USER [IF EXISTS] name [,...] * DROP ROLE [IF EXISTS] name [,...] @@ -22,11 +23,13 @@ public: EntityType type; bool if_exists = false; Strings names; - std::vector row_policies_name_parts; + std::shared_ptr row_policy_names; String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } + + void replaceEmptyDatabaseWithCurrent(const String & current_database) const; }; } diff --git a/src/Parsers/ASTRowPolicyName.cpp b/src/Parsers/ASTRowPolicyName.cpp new file mode 100644 index 00000000000..5e3c494ccd3 --- /dev/null +++ b/src/Parsers/ASTRowPolicyName.cpp @@ -0,0 +1,134 @@ +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + +void ASTRowPolicyName::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + const String & database = name_parts.database; + const String & table_name = name_parts.table_name; + const String & short_name = name_parts.short_name; + settings.ostr << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " + << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") + << backQuoteIfNeed(table_name); + + formatOnCluster(settings); +} + + +void ASTRowPolicyName::replaceEmptyDatabaseWithCurrent(const String & current_database) +{ + if (name_parts.database.empty()) + name_parts.database = current_database; +} + + +void ASTRowPolicyNames::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + if (name_parts.empty()) + throw Exception("No names of row policies in AST", ErrorCodes::LOGICAL_ERROR); + + bool same_short_name = true; + if (name_parts.size() > 1) + { + for (size_t i = 1; i != name_parts.size(); ++i) + if (name_parts[i].short_name != name_parts[0].short_name) + { + same_short_name = false; + break; + } + } + + bool same_db_and_table_name = true; + if (name_parts.size() > 1) + { + for (size_t i = 1; i != name_parts.size(); ++i) + if ((name_parts[i].database != name_parts[0].database) || (name_parts[i].table_name != name_parts[0].table_name)) + { + same_db_and_table_name = false; + break; + } + } + + if (same_short_name) + { + const String & short_name = name_parts[0].short_name; + settings.ostr << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " + << (settings.hilite ? hilite_none : ""); + + bool need_comma = false; + for (const auto & np : name_parts) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + const String & database = np.database; + const String & table_name = np.table_name; + if (!database.empty()) + settings.ostr << backQuoteIfNeed(database) + "."; + settings.ostr << backQuoteIfNeed(table_name); + } + } + else if (same_db_and_table_name) + { + bool need_comma = false; + for (const auto & np : name_parts) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + const String & short_name = np.short_name; + settings.ostr << backQuoteIfNeed(short_name); + } + + const String & database = name_parts[0].database; + const String & table_name = name_parts[0].table_name; + settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); + if (!database.empty()) + settings.ostr << backQuoteIfNeed(database) + "."; + settings.ostr << backQuoteIfNeed(table_name); + } + else + { + bool need_comma = false; + for (const auto & np : name_parts) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + const String & short_name = np.short_name; + const String & database = np.database; + const String & table_name = np.table_name; + settings.ostr << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " + << (settings.hilite ? hilite_none : ""); + if (!database.empty()) + settings.ostr << backQuoteIfNeed(database) + "."; + settings.ostr << backQuoteIfNeed(table_name); + } + } + + formatOnCluster(settings); +} + + +Strings ASTRowPolicyNames::toStrings() const +{ + Strings res; + res.reserve(name_parts.size()); + for (const auto & np : name_parts) + res.emplace_back(np.toString()); + return res; +} + + +void ASTRowPolicyNames::replaceEmptyDatabaseWithCurrent(const String & current_database) +{ + for (auto & np : name_parts) + if (np.database.empty()) + np.database = current_database; +} + +} diff --git a/src/Parsers/ASTRowPolicyName.h b/src/Parsers/ASTRowPolicyName.h new file mode 100644 index 00000000000..ac2f84f5d8b --- /dev/null +++ b/src/Parsers/ASTRowPolicyName.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ +/** Represents a row policy's name in one of the following forms: + * short_name ON [db.]table_name [ON CLUSTER 'cluster_name'] + * short_name [ON CLUSTER 'cluster_name'] ON [db.]table_name + */ +class ASTRowPolicyName : public IAST, public ASTQueryWithOnCluster +{ +public: + RowPolicy::NameParts name_parts; + String toString() const { return name_parts.getName(); } + + String getID(char) const override { return "RowPolicyName"; } + ASTPtr clone() const override { return std::make_shared(*this); } + void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } + + void replaceEmptyDatabaseWithCurrent(const String & current_database); +}; + + +/** Represents multiple names of row policies, comma-separated, in one of the following forms: + * short_name1 ON [db1.]table_name1 [, short_name2 ON [db2.]table_name2 ...] [ON CLUSTER 'cluster_name'] + * short_name1 [, short_name2 ...] ON [db.]table_name [ON CLUSTER 'cluster_name'] + * short_name1 [, short_name2 ...] [ON CLUSTER 'cluster_name'] ON [db.]table_name + * short_name ON [db1.]table_name1 [, [db2.]table_name2 ...] [ON CLUSTER 'cluster_name'] + * short_name [ON CLUSTER 'cluster_name'] ON [db1.]table_name1 [, [db2.]table_name2 ...] + */ +class ASTRowPolicyNames : public IAST, public ASTQueryWithOnCluster +{ +public: + std::vector name_parts; + Strings toStrings() const; + + String getID(char) const override { return "RowPolicyNames"; } + ASTPtr clone() const override { return std::make_shared(*this); } + void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } + + void replaceEmptyDatabaseWithCurrent(const String & current_database); +}; +} diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp index f81b30428fb..cbd31d0d53c 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp @@ -1,10 +1,25 @@ #include +#include #include namespace DB { -using EntityTypeInfo = IAccessEntity::TypeInfo; +namespace +{ + using EntityTypeInfo = IAccessEntity::TypeInfo; + + void formatNames(const Strings & names, const IAST::FormatSettings & settings) + { + bool need_comma = false; + for (const auto & name : names) + { + if (std::exchange(need_comma, true)) + settings.ostr << ','; + settings.ostr << ' ' << backQuoteIfNeed(name); + } + } +} String ASTShowCreateAccessEntityQuery::getID(char) const @@ -30,14 +45,18 @@ void ASTShowCreateAccessEntityQuery::formatQueryImpl(const FormatSettings & sett } else if (type == EntityType::ROW_POLICY) { - const String & database = row_policy_name_parts.database; - const String & table_name = row_policy_name_parts.table_name; - const String & short_name = row_policy_name_parts.short_name; - settings.ostr << ' ' << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " - << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") - << backQuoteIfNeed(table_name); + settings.ostr << " "; + row_policy_names->format(settings); } else - settings.ostr << " " << backQuoteIfNeed(name); + formatNames(names, settings); } + + +void ASTShowCreateAccessEntityQuery::replaceEmptyDatabaseWithCurrent(const String & current_database) +{ + if (row_policy_names) + row_policy_names->replaceEmptyDatabaseWithCurrent(current_database); +} + } diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.h b/src/Parsers/ASTShowCreateAccessEntityQuery.h index df7be2e257c..5fd16d622f7 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.h +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.h @@ -1,12 +1,14 @@ #pragma once #include -#include +#include namespace DB { -/** SHOW CREATE QUOTA [name | CURRENT] +class ASTRowPolicyNames; + +/** SHOW CREATE QUOTA [name] * SHOW CREATE [ROW] POLICY name ON [database.]table * SHOW CREATE USER [name | CURRENT_USER] * SHOW CREATE ROLE name @@ -18,14 +20,16 @@ public: using EntityType = IAccessEntity::Type; EntityType type; - String name; + Strings names; bool current_quota = false; bool current_user = false; - RowPolicy::NameParts row_policy_name_parts; + std::shared_ptr row_policy_names; String getID(char) const override; ASTPtr clone() const override; + void replaceEmptyDatabaseWithCurrent(const String & current_database); + protected: void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; diff --git a/src/Parsers/ASTUserNameWithHost.cpp b/src/Parsers/ASTUserNameWithHost.cpp new file mode 100644 index 00000000000..13d34b99b3d --- /dev/null +++ b/src/Parsers/ASTUserNameWithHost.cpp @@ -0,0 +1,74 @@ +#include +#include + + +namespace DB +{ + +void ASTUserNameWithHost::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + settings.ostr << backQuoteIfNeed(base_name); + + if (!host_pattern.empty()) + settings.ostr << "@" << backQuoteIfNeed(host_pattern); +} + +String ASTUserNameWithHost::toString() const +{ + String res = base_name; + if (!host_pattern.empty()) + res += '@' + host_pattern; + return res; +} + +void ASTUserNameWithHost::concatParts() +{ + base_name = toString(); + host_pattern.clear(); +} + + +void ASTUserNamesWithHost::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + assert(!names.empty()); + bool need_comma = false; + for (const auto & name : names) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + name->format(settings); + } +} + +Strings ASTUserNamesWithHost::toStrings() const +{ + Strings res; + res.reserve(names.size()); + for (const auto & name : names) + res.emplace_back(name->toString()); + return res; +} + +void ASTUserNamesWithHost::concatParts() +{ + for (auto & name : names) + name->concatParts(); +} + + +bool ASTUserNamesWithHost::getHostPatternIfCommon(String & out_common_host_pattern) const +{ + out_common_host_pattern.clear(); + + if (names.empty()) + return true; + + for (size_t i = 1; i != names.size(); ++i) + if (names[i]->host_pattern != names[0]->host_pattern) + return false; + + out_common_host_pattern = names[0]->host_pattern; + return true; +} + +} diff --git a/src/Parsers/ASTUserNameWithHost.h b/src/Parsers/ASTUserNameWithHost.h new file mode 100644 index 00000000000..00b1570e062 --- /dev/null +++ b/src/Parsers/ASTUserNameWithHost.h @@ -0,0 +1,53 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Represents a user name. + * It can be a simple string or identifier or something like `name@host`. + * In the last case `host` specifies the hosts user is allowed to connect from. + * The `host` can be an ip address, ip subnet, or a host name. + * The % and _ wildcard characters are permitted in `host`. + * These have the same meaning as for pattern-matching operations performed with the LIKE operator. + */ +class ASTUserNameWithHost : public IAST +{ +public: + String base_name; + String host_pattern; + + String toString() const; + void concatParts(); + + ASTUserNameWithHost() = default; + ASTUserNameWithHost(const String & name_) : base_name(name_) {} + String getID(char) const override { return "UserNameWithHost"; } + ASTPtr clone() const override { return std::make_shared(*this); } + void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; +}; + + +class ASTUserNamesWithHost : public IAST +{ +public: + std::vector> names; + + size_t size() const { return names.size(); } + auto begin() const { return names.begin(); } + auto end() const { return names.end(); } + auto front() const { return *begin(); } + void push_back(const String & name_) { names.push_back(std::make_shared(name_)); } + + Strings toStrings() const; + void concatParts(); + bool getHostPatternIfCommon(String & out_common_host_pattern) const; + + String getID(char) const override { return "UserNamesWithHost"; } + ASTPtr clone() const override { return std::make_shared(*this); } + void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; +}; + +} diff --git a/src/Parsers/ParserCreateQuotaQuery.cpp b/src/Parsers/ParserCreateQuotaQuery.cpp index b505fd25a95..5eb138d6be0 100644 --- a/src/Parsers/ParserCreateQuotaQuery.cpp +++ b/src/Parsers/ParserCreateQuotaQuery.cpp @@ -240,8 +240,8 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe or_replace = true; } - String name; - if (!parseIdentifierOrStringLiteral(pos, expected, name)) + Strings names; + if (!parseIdentifiersOrStringLiterals(pos, expected, names)) return false; String new_name; @@ -251,7 +251,7 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe while (true) { - if (alter && new_name.empty() && parseRenameTo(pos, expected, new_name)) + if (alter && new_name.empty() && (names.size() == 1) && parseRenameTo(pos, expected, new_name)) continue; if (!key_type && parseKeyType(pos, expected, key_type)) @@ -280,7 +280,7 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe query->if_not_exists = if_not_exists; query->or_replace = or_replace; query->cluster = std::move(cluster); - query->name = std::move(name); + query->names = std::move(names); query->new_name = std::move(new_name); query->key_type = key_type; query->all_limits = std::move(all_limits); diff --git a/src/Parsers/ParserCreateRoleQuery.cpp b/src/Parsers/ParserCreateRoleQuery.cpp index 2a6f2dd2c90..08dd31c51a3 100644 --- a/src/Parsers/ParserCreateRoleQuery.cpp +++ b/src/Parsers/ParserCreateRoleQuery.cpp @@ -84,8 +84,8 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec or_replace = true; } - String name; - if (!parseRoleName(pos, expected, name)) + Strings names; + if (!parseRoleNames(pos, expected, names)) return false; String new_name; @@ -94,7 +94,7 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec while (true) { - if (alter && parseRenameTo(pos, expected, new_name)) + if (alter && new_name.empty() && (names.size() == 1) && parseRenameTo(pos, expected, new_name)) continue; if (parseSettings(pos, expected, attach_mode, settings)) @@ -115,7 +115,7 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->if_not_exists = if_not_exists; query->or_replace = or_replace; query->cluster = std::move(cluster); - query->name = std::move(name); + query->names = std::move(names); query->new_name = std::move(new_name); query->settings = std::move(settings); diff --git a/src/Parsers/ParserCreateRowPolicyQuery.cpp b/src/Parsers/ParserCreateRowPolicyQuery.cpp index 2456cb80368..4f5f2989a7b 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/ParserCreateRowPolicyQuery.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -227,22 +229,22 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & or_replace = true; } - RowPolicy::NameParts name_parts; - String & database = name_parts.database; - String & table_name = name_parts.table_name; - String & short_name = name_parts.short_name; - if (!parseIdentifierOrStringLiteral(pos, expected, short_name) || !ParserKeyword{"ON"}.ignore(pos, expected) - || !parseDatabaseAndTableName(pos, expected, database, table_name)) + ParserRowPolicyNames names_parser; + names_parser.allowOnCluster(); + ASTPtr names_ast; + if (!names_parser.parse(pos, names_ast, expected)) return false; + auto names = typeid_cast>(names_ast); + String cluster = std::exchange(names->cluster, ""); + String new_short_name; std::optional is_restrictive; std::array, MAX_CONDITION_TYPE> conditions; - String cluster; while (true) { - if (alter && new_short_name.empty() && parseRenameTo(pos, expected, new_short_name)) + if (alter && new_short_name.empty() && (names->name_parts.size() == 1) && parseRenameTo(pos, expected, new_short_name)) continue; if (!is_restrictive && parseAsRestrictiveOrPermissive(pos, expected, is_restrictive)) @@ -272,7 +274,7 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & query->if_not_exists = if_not_exists; query->or_replace = or_replace; query->cluster = std::move(cluster); - query->name_parts = std::move(name_parts); + query->names = std::move(names); query->new_short_name = std::move(new_short_name); query->is_restrictive = is_restrictive; query->conditions = std::move(conditions); diff --git a/src/Parsers/ParserCreateSettingsProfileQuery.cpp b/src/Parsers/ParserCreateSettingsProfileQuery.cpp index 83d0f0c1d91..4d5a9c09ad8 100644 --- a/src/Parsers/ParserCreateSettingsProfileQuery.cpp +++ b/src/Parsers/ParserCreateSettingsProfileQuery.cpp @@ -100,8 +100,8 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec or_replace = true; } - String name; - if (!parseIdentifierOrStringLiteral(pos, expected, name)) + Strings names; + if (!parseIdentifiersOrStringLiterals(pos, expected, names)) return false; String new_name; @@ -110,7 +110,7 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec while (true) { - if (alter && parseRenameTo(pos, expected, new_name)) + if (alter && new_name.empty() && (names.size() == 1) && parseRenameTo(pos, expected, new_name)) continue; if (parseSettings(pos, expected, attach_mode, settings)) @@ -137,7 +137,7 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec query->if_not_exists = if_not_exists; query->or_replace = or_replace; query->cluster = std::move(cluster); - query->name = std::move(name); + query->names = std::move(names); query->new_name = std::move(new_name); query->settings = std::move(settings); query->to_roles = std::move(to_roles); diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index e03f8334d42..e99457a2f87 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -16,11 +18,6 @@ namespace DB { -namespace ErrorCodes -{ -} - - namespace { bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_name) @@ -268,10 +265,11 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec or_replace = true; } - String name; - std::optional host_pattern; - if (!parseUserName(pos, expected, name, host_pattern)) + ASTPtr names_ast; + if (!ParserUserNamesWithHost{}.parse(pos, names_ast, expected)) return false; + auto names = typeid_cast>(names_ast); + auto names_ref = names->names; String new_name; std::optional authentication; @@ -301,7 +299,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (alter) { - if (new_name.empty() && parseRenameTo(pos, expected, new_name)) + if (new_name.empty() && (names->size() == 1) && parseRenameTo(pos, expected, new_name)) continue; if (parseHosts(pos, expected, "ADD", add_hosts) || parseHosts(pos, expected, "DROP", remove_hosts)) @@ -311,8 +309,17 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec break; } - if (!alter && !hosts && host_pattern) - hosts.emplace().addLikePattern(*host_pattern); + if (!alter && !hosts) + { + String common_host_pattern; + if (names->getHostPatternIfCommon(common_host_pattern) && !common_host_pattern.empty()) + { + hosts.emplace().addLikePattern(common_host_pattern); + names->concatParts(); + } + } + else if (alter) + names->concatParts(); auto query = std::make_shared(); node = query; @@ -323,7 +330,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->if_not_exists = if_not_exists; query->or_replace = or_replace; query->cluster = std::move(cluster); - query->name = std::move(name); + query->names = std::move(names); query->new_name = std::move(new_name); query->authentication = std::move(authentication); query->hosts = std::move(hosts); diff --git a/src/Parsers/ParserDropAccessEntityQuery.cpp b/src/Parsers/ParserDropAccessEntityQuery.cpp index 15f8bbf0a62..d5eac710631 100644 --- a/src/Parsers/ParserDropAccessEntityQuery.cpp +++ b/src/Parsers/ParserDropAccessEntityQuery.cpp @@ -1,8 +1,9 @@ #include #include #include +#include +#include #include -#include #include #include @@ -14,65 +15,11 @@ namespace using EntityType = IAccessEntity::Type; using EntityTypeInfo = IAccessEntity::TypeInfo; - bool parseNames(IParserBase::Pos & pos, Expected & expected, Strings & names) + bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster) { return IParserBase::wrapParseImpl(pos, [&] { - Strings res_names; - do - { - String name; - if (!parseIdentifierOrStringLiteral(pos, expected, name)) - return false; - - res_names.push_back(std::move(name)); - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); - - names = std::move(res_names); - return true; - }); - } - - bool parseRowPolicyNames(IParserBase::Pos & pos, Expected & expected, std::vector & name_parts) - { - return IParserBase::wrapParseImpl(pos, [&] - { - std::vector res_name_parts; - do - { - Strings short_names; - if (!parseNames(pos, expected, short_names)) - return false; - String database, table_name; - if (!ParserKeyword{"ON"}.ignore(pos, expected) || !parseDatabaseAndTableName(pos, expected, database, table_name)) - return false; - for (String & short_name : short_names) - res_name_parts.push_back({std::move(short_name), database, table_name}); - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); - - name_parts = std::move(res_name_parts); - return true; - }); - } - - bool parseUserNames(IParserBase::Pos & pos, Expected & expected, Strings & names) - { - return IParserBase::wrapParseImpl(pos, [&] - { - Strings res_names; - do - { - String name; - if (!parseUserName(pos, expected, name)) - return false; - - res_names.emplace_back(std::move(name)); - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); - names = std::move(res_names); - return true; + return ParserKeyword{"ON"}.ignore(pos, expected) && ASTQueryWithOnCluster::parse(pos, cluster, expected); }); } } @@ -101,7 +48,8 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & if_exists = true; Strings names; - std::vector row_policies_name_parts; + std::shared_ptr row_policy_names; + String cluster; if ((type == EntityType::USER) || (type == EntityType::ROLE)) { @@ -110,21 +58,22 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & } else if (type == EntityType::ROW_POLICY) { - if (!parseRowPolicyNames(pos, expected, row_policies_name_parts)) + ParserRowPolicyNames parser; + ASTPtr ast; + parser.allowOnCluster(); + if (!parser.parse(pos, ast, expected)) return false; + row_policy_names = typeid_cast>(ast); + cluster = std::exchange(row_policy_names->cluster, ""); } else { - if (!parseNames(pos, expected, names)) + if (!parseIdentifiersOrStringLiterals(pos, expected, names)) return false; } - String cluster; - if (ParserKeyword{"ON"}.ignore(pos, expected)) - { - if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) - return false; - } + if (cluster.empty()) + parseOnCluster(pos, expected, cluster); auto query = std::make_shared(); node = query; @@ -133,7 +82,7 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & query->if_exists = if_exists; query->cluster = std::move(cluster); query->names = std::move(names); - query->row_policies_name_parts = std::move(row_policies_name_parts); + query->row_policy_names = std::move(row_policy_names); return true; } diff --git a/src/Parsers/ParserRowPolicyName.cpp b/src/Parsers/ParserRowPolicyName.cpp new file mode 100644 index 00000000000..8f1ef91f7c1 --- /dev/null +++ b/src/Parsers/ParserRowPolicyName.cpp @@ -0,0 +1,154 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ + bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster) + { + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{"ON"}.ignore(pos, expected) && ASTQueryWithOnCluster::parse(pos, cluster, expected); + }); + } + + + bool parseOnDatabaseAndTableName(IParser::Pos & pos, Expected & expected, String & database, String & table_name) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (!ParserKeyword{"ON"}.ignore(pos, expected)) + return false; + + return parseDatabaseAndTableName(pos, expected, database, table_name); + }); + } + + + bool parseOnDatabaseAndTableName(IParser::Pos & pos, Expected & expected, std::pair & database_and_table_name) + { + return parseOnDatabaseAndTableName(pos, expected, database_and_table_name.first, database_and_table_name.second); + } + + + bool parseOnDatabaseAndTableNames(IParser::Pos & pos, Expected & expected, std::vector> & database_and_table_names) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (!ParserKeyword{"ON"}.ignore(pos, expected)) + return false; + + std::vector> res; + std::optional pos_before_comma; + do + { + String database, table_name; + if (!parseDatabaseAndTableName(pos, expected, database, table_name)) + return false; + + String unused; + if (pos_before_comma && database.empty() && ParserKeyword{"ON"}.ignore(pos, expected) + && !ASTQueryWithOnCluster::parse(pos, unused, expected)) + { + pos = *pos_before_comma; + break; + } + + res.emplace_back(std::move(database), std::move(table_name)); + pos_before_comma = pos; + } + while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + database_and_table_names = std::move(res); + return true; + }); + } +} + + +bool ParserRowPolicyName::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + RowPolicy::NameParts name_parts; + if (!parseIdentifierOrStringLiteral(pos, expected, name_parts.short_name)) + return false; + + String cluster; + parseOnCluster(pos, expected, cluster); + + if (!parseOnDatabaseAndTableName(pos, expected, name_parts.database, name_parts.table_name)) + return false; + + if (cluster.empty()) + parseOnCluster(pos, expected, cluster); + + auto result = std::make_shared(); + result->name_parts = std::move(name_parts); + result->cluster = std::move(cluster); + node = result; + return true; +} + + +bool ParserRowPolicyNames::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + std::vector name_parts; + String cluster; + + do + { + std::vector short_names; + bool allowed_multiple_short_names = name_parts.empty(); + if (allowed_multiple_short_names) + { + if (!parseIdentifiersOrStringLiterals(pos, expected, short_names)) + return false; + } + else + { + if (!parseIdentifierOrStringLiteral(pos, expected, short_names.emplace_back())) + return false; + } + + bool allowed_on_cluster = allow_on_cluster && name_parts.empty(); + if (allowed_on_cluster) + parseOnCluster(pos, expected, cluster); + + std::vector> database_and_table_names; + bool allowed_multiple_db_and_table_names = ((name_parts.empty()) && (short_names.size() == 1)); + if (allowed_multiple_db_and_table_names) + { + if (!parseOnDatabaseAndTableNames(pos, expected, database_and_table_names)) + return false; + } + else + { + if (!parseOnDatabaseAndTableName(pos, expected, database_and_table_names.emplace_back())) + return false; + } + + allowed_on_cluster &= cluster.empty(); + if (allowed_on_cluster) + parseOnCluster(pos, expected, cluster); + + for (const String & short_name : short_names) + for (const auto & [database, table_name] : database_and_table_names) + name_parts.push_back({short_name, database, table_name}); + + if ((short_names.size() != 1) || (database_and_table_names.size() != 1) || !cluster.empty()) + break; + } + while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + + auto result = std::make_shared(); + result->name_parts = std::move(name_parts); + result->cluster = std::move(cluster); + node = result; + return true; +} + +} diff --git a/src/Parsers/ParserRowPolicyName.h b/src/Parsers/ParserRowPolicyName.h new file mode 100644 index 00000000000..6af0519d161 --- /dev/null +++ b/src/Parsers/ParserRowPolicyName.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + + +namespace DB +{ +/** Parses a string in one of the following form: + * short_name ON [db.]table_name [ON CLUSTER 'cluster_name'] + * short_name [ON CLUSTER 'cluster_name'] ON [db.]table_name + */ +class ParserRowPolicyName : public IParserBase +{ +public: + void allowOnCluster(bool allow = true) { allow_on_cluster = allow; } + +protected: + const char * getName() const override { return "RowPolicyName"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +private: + bool allow_on_cluster = false; +}; + + +/** Parses a string in one of the following form: + * short_name1 ON [db1.]table_name1 [, short_name2 ON [db2.]table_name2 ...] [ON CLUSTER 'cluster_name'] + * short_name1 [, short_name2 ...] ON [db.]table_name [ON CLUSTER 'cluster_name'] + * short_name1 [, short_name2 ...] [ON CLUSTER 'cluster_name'] ON [db.]table_name + * short_name ON [db1.]table_name1 [, [db2.]table_name2 ...] [ON CLUSTER 'cluster_name'] + * short_name [ON CLUSTER 'cluster_name'] ON [db1.]table_name1 [, [db2.]table_name2 ...] + */ +class ParserRowPolicyNames : public IParserBase +{ +public: + void allowOnCluster(bool allow = true) { allow_on_cluster = allow; } + +protected: + const char * getName() const override { return "SettingsProfileElements"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +private: + bool allow_on_cluster = false; +}; + +} diff --git a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp index ca520f4df6f..465a6c380b1 100644 --- a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp @@ -2,7 +2,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -32,33 +33,33 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe if (!type) return false; - String name; + Strings names; bool current_quota = false; bool current_user = false; - RowPolicy::NameParts row_policy_name_parts; + std::shared_ptr row_policy_names; if (type == EntityType::USER) { - if (!parseUserNameOrCurrentUserTag(pos, expected, name, current_user)) + if (parseCurrentUserTag(pos, expected)) current_user = true; + else if (!parseUserNames(pos, expected, names)) + return false; } else if (type == EntityType::ROLE) { - if (!parseRoleName(pos, expected, name)) + if (!parseRoleNames(pos, expected, names)) return false; } else if (type == EntityType::ROW_POLICY) { - String & database = row_policy_name_parts.database; - String & table_name = row_policy_name_parts.table_name; - String & short_name = row_policy_name_parts.short_name; - if (!parseIdentifierOrStringLiteral(pos, expected, short_name) || !ParserKeyword{"ON"}.ignore(pos, expected) - || !parseDatabaseAndTableName(pos, expected, database, table_name)) + ASTPtr ast; + if (!ParserRowPolicyNames{}.parse(pos, ast, expected)) return false; + row_policy_names = typeid_cast>(ast); } else if (type == EntityType::QUOTA) { - if (!parseIdentifierOrStringLiteral(pos, expected, name)) + if (!parseIdentifiersOrStringLiterals(pos, expected, names)) { /// SHOW CREATE QUOTA current_quota = true; @@ -66,7 +67,7 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe } else if (type == EntityType::SETTINGS_PROFILE) { - if (!parseIdentifierOrStringLiteral(pos, expected, name)) + if (!parseIdentifiersOrStringLiterals(pos, expected, names)) return false; } @@ -74,10 +75,10 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe node = query; query->type = *type; - query->name = std::move(name); + query->names = std::move(names); query->current_quota = current_quota; query->current_user = current_user; - query->row_policy_name_parts = std::move(row_policy_name_parts); + query->row_policy_names = std::move(row_policy_names); return true; } diff --git a/src/Parsers/ParserShowGrantsQuery.cpp b/src/Parsers/ParserShowGrantsQuery.cpp index c667894f1d7..993346d2eeb 100644 --- a/src/Parsers/ParserShowGrantsQuery.cpp +++ b/src/Parsers/ParserShowGrantsQuery.cpp @@ -16,7 +16,9 @@ bool ParserShowGrantsQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (ParserKeyword{"FOR"}.ignore(pos, expected)) { - if (!parseUserNameOrCurrentUserTag(pos, expected, name, current_user)) + if (parseCurrentUserTag(pos, expected)) + current_user = true; + else if (!parseUserName(pos, expected, name)) return false; } else diff --git a/src/Parsers/ParserUserNameWithHost.cpp b/src/Parsers/ParserUserNameWithHost.cpp new file mode 100644 index 00000000000..19ec7a9bbd1 --- /dev/null +++ b/src/Parsers/ParserUserNameWithHost.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ +bool ParserUserNameWithHost::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + String base_name; + if (!parseIdentifierOrStringLiteral(pos, expected, base_name)) + return false; + + boost::algorithm::trim(base_name); + + String host_pattern; + if (ParserToken{TokenType::At}.ignore(pos, expected)) + { + if (!parseIdentifierOrStringLiteral(pos, expected, host_pattern)) + return false; + + boost::algorithm::trim(host_pattern); + if (host_pattern == "%") + host_pattern.clear(); + } + + auto result = std::make_shared(); + result->base_name = std::move(base_name); + result->host_pattern = std::move(host_pattern); + node = result; + return true; +} + + +bool ParserUserNamesWithHost::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + std::vector> names; + do + { + ASTPtr ast; + if (!ParserUserNameWithHost{}.parse(pos, ast, expected)) + return false; + + names.emplace_back(typeid_cast>(ast)); + } + while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + + auto result = std::make_shared(); + result->names = std::move(names); + node = result; + return true; +} + +} diff --git a/src/Parsers/ParserUserNameWithHost.h b/src/Parsers/ParserUserNameWithHost.h new file mode 100644 index 00000000000..453b816a98d --- /dev/null +++ b/src/Parsers/ParserUserNameWithHost.h @@ -0,0 +1,26 @@ +#pragma once + +#include + + +namespace DB +{ +/** Parses a user name. + * It can be a simple string or identifier or something like `name@host`. + */ +class ParserUserNameWithHost : public IParserBase +{ +protected: + const char * getName() const override { return "UserNameWithHost"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + + +class ParserUserNamesWithHost : public IParserBase +{ +protected: + const char * getName() const override { return "UserNamesWithHost"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Parsers/parseIdentifierOrStringLiteral.cpp b/src/Parsers/parseIdentifierOrStringLiteral.cpp index 7258d3e39da..22c77af0b09 100644 --- a/src/Parsers/parseIdentifierOrStringLiteral.cpp +++ b/src/Parsers/parseIdentifierOrStringLiteral.cpp @@ -3,6 +3,7 @@ #include "ExpressionElementParsers.h" #include "ASTLiteral.h" #include "ASTIdentifier.h" +#include #include namespace DB @@ -25,4 +26,25 @@ bool parseIdentifierOrStringLiteral(IParser::Pos & pos, Expected & expected, Str return true; } + +bool parseIdentifiersOrStringLiterals(IParser::Pos & pos, Expected & expected, Strings & result) +{ + return IParserBase::wrapParseImpl(pos, [&] + { + Strings strs; + do + { + String str; + if (!parseIdentifierOrStringLiteral(pos, expected, str)) + return false; + + strs.push_back(std::move(str)); + } + while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + + result = std::move(strs); + return true; + }); +} + } diff --git a/src/Parsers/parseIdentifierOrStringLiteral.h b/src/Parsers/parseIdentifierOrStringLiteral.h index 0f49d44ee01..0174c481704 100644 --- a/src/Parsers/parseIdentifierOrStringLiteral.h +++ b/src/Parsers/parseIdentifierOrStringLiteral.h @@ -9,4 +9,7 @@ namespace DB * name, `name` or 'name' */ bool parseIdentifierOrStringLiteral(IParser::Pos & pos, Expected & expected, String & result); +/** Parse a list of identifiers or string literals. */ +bool parseIdentifiersOrStringLiterals(IParser::Pos & pos, Expected & expected, Strings & result); + } diff --git a/src/Parsers/parseUserName.cpp b/src/Parsers/parseUserName.cpp index e6b91ba4af3..1f25f51ef22 100644 --- a/src/Parsers/parseUserName.cpp +++ b/src/Parsers/parseUserName.cpp @@ -1,64 +1,46 @@ #include -#include +#include +#include #include -#include namespace DB { -bool parseUserName(IParser::Pos & pos, Expected & expected, String & user_name, std::optional & host_like_pattern) + +bool parseUserName(IParser::Pos & pos, Expected & expected, String & user_name) { - String name; - if (!parseIdentifierOrStringLiteral(pos, expected, name)) + ASTPtr ast; + if (!ParserUserNameWithHost{}.parse(pos, ast, expected)) return false; - - boost::algorithm::trim(name); - - std::optional pattern; - if (ParserToken{TokenType::At}.ignore(pos, expected)) - { - if (!parseIdentifierOrStringLiteral(pos, expected, pattern.emplace())) - return false; - - boost::algorithm::trim(*pattern); - } - - if (pattern && (pattern != "%")) - name += '@' + *pattern; - - user_name = std::move(name); - host_like_pattern = std::move(pattern); + user_name = ast->as().toString(); return true; } -bool parseUserName(IParser::Pos & pos, Expected & expected, String & user_name) +bool parseUserNames(IParser::Pos & pos, Expected & expected, Strings & user_names) { - std::optional unused_pattern; - return parseUserName(pos, expected, user_name, unused_pattern); + ASTPtr ast; + if (!ParserUserNamesWithHost{}.parse(pos, ast, expected)) + return false; + user_names = ast->as().toStrings(); + return true; } -bool parseUserNameOrCurrentUserTag(IParser::Pos & pos, Expected & expected, String & user_name, bool & current_user) +bool parseCurrentUserTag(IParser::Pos & pos, Expected & expected) { - if (ParserKeyword{"CURRENT_USER"}.ignore(pos, expected) || ParserKeyword{"currentUser"}.ignore(pos, expected)) + return IParserBase::wrapParseImpl(pos, [&] { + if (!ParserKeyword{"CURRENT_USER"}.ignore(pos, expected) && !ParserKeyword{"currentUser"}.ignore(pos, expected)) + return false; + if (ParserToken{TokenType::OpeningRoundBracket}.ignore(pos, expected)) { if (!ParserToken{TokenType::ClosingRoundBracket}.ignore(pos, expected)) return false; } - current_user = true; return true; - } - - if (parseUserName(pos, expected, user_name)) - { - current_user = false; - return true; - } - - return false; + }); } } diff --git a/src/Parsers/parseUserName.h b/src/Parsers/parseUserName.h index 641aa09d1f3..c1ad36c936e 100644 --- a/src/Parsers/parseUserName.h +++ b/src/Parsers/parseUserName.h @@ -10,11 +10,15 @@ namespace DB /// The `host` can be an ip address, ip subnet, or a host name. /// The % and _ wildcard characters are permitted in `host`. /// These have the same meaning as for pattern-matching operations performed with the LIKE operator. -bool parseUserName(IParser::Pos & pos, Expected & expected, String & user_name, std::optional & host_like_pattern); bool parseUserName(IParser::Pos & pos, Expected & expected, String & user_name); -/// Parses either a user name or the 'CURRENT_USER' keyword (or some of the aliases). -bool parseUserNameOrCurrentUserTag(IParser::Pos & pos, Expected & expected, String & user_name, bool & current_user); +/// Parses a comma-separated list of user names. +bool parseUserNames(IParser::Pos & pos, Expected & expected, Strings & user_names); + + +/// Parses either the 'CURRENT_USER' keyword (or some of its aliases). +bool parseCurrentUserTag(IParser::Pos & pos, Expected & expected); + /// Parses a role name. It follows the same rules as a user name, but allowed hosts are never checked /// (because roles are not used to connect to server). @@ -22,4 +26,11 @@ inline bool parseRoleName(IParser::Pos & pos, Expected & expected, String & role { return parseUserName(pos, expected, role_name); } + +/// Parses a comma-separated list of role names. +inline bool parseRoleNames(IParser::Pos & pos, Expected & expected, Strings & role_names) +{ + return parseUserNames(pos, expected, role_names); +} + } diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index 8c7e4ff68af..60672c0c116 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -37,6 +37,7 @@ SRCS( ASTQueryWithOnCluster.cpp ASTQueryWithOutput.cpp ASTQueryWithTableAndOutput.cpp + ASTRowPolicyName.cpp ASTSampleRatio.cpp ASTSelectQuery.cpp ASTSelectWithUnionQuery.cpp @@ -51,6 +52,7 @@ SRCS( ASTSystemQuery.cpp ASTTablesInSelectQuery.cpp ASTTTLElement.cpp + ASTUserNameWithHost.cpp ASTWithAlias.cpp CommonParsers.cpp ExpressionElementParsers.cpp @@ -88,6 +90,7 @@ SRCS( ParserQuery.cpp ParserQueryWithOutput.cpp ParserRenameQuery.cpp + ParserRowPolicyName.cpp ParserSampleRatio.cpp ParserSelectQuery.cpp ParserSelectWithUnionQuery.cpp @@ -104,6 +107,7 @@ SRCS( ParserTablesInSelectQuery.cpp ParserUnionQueryElement.cpp ParserUseQuery.cpp + ParserUserNameWithHost.cpp ParserWatchQuery.cpp parseUserName.cpp queryToString.cpp diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index 4dc9baca0a0..07c72e94be0 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -38,4 +38,3 @@ def test_access_control_on_cluster(): assert "There is no user `Alex`" in ch1.query_and_get_error("SHOW CREATE USER Alex") assert "There is no user `Alex`" in ch2.query_and_get_error("SHOW CREATE USER Alex") assert "There is no user `Alex`" in ch3.query_and_get_error("SHOW CREATE USER Alex") - From 92b9f4a88d6b79cc7626376d50b02a3a7985544d Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 30 May 2020 23:10:45 +0300 Subject: [PATCH 225/329] Rename ExtendedRoleSet => RolesOrUsersSet. --- src/Access/DiskAccessStorage.cpp | 30 ++++---- src/Access/Quota.h | 4 +- src/Access/QuotaCache.h | 2 +- ...xtendedRoleSet.cpp => RolesOrUsersSet.cpp} | 69 +++++++++---------- .../{ExtendedRoleSet.h => RolesOrUsersSet.h} | 42 +++++------ src/Access/RowPolicy.h | 4 +- src/Access/RowPolicyCache.h | 2 +- src/Access/SettingsProfile.h | 4 +- src/Access/User.h | 4 +- src/Access/ya.make | 2 +- .../InterpreterCreateQuotaQuery.cpp | 8 +-- .../InterpreterCreateRowPolicyQuery.cpp | 8 +-- .../InterpreterCreateSettingsProfileQuery.cpp | 8 +-- .../InterpreterCreateUserQuery.cpp | 10 +-- src/Interpreters/InterpreterGrantQuery.cpp | 12 ++-- src/Interpreters/InterpreterSetRoleQuery.cpp | 12 ++-- src/Interpreters/InterpreterSetRoleQuery.h | 4 +- ...InterpreterShowCreateAccessEntityQuery.cpp | 4 +- .../InterpreterShowGrantsQuery.cpp | 8 +-- src/Parsers/ASTCreateQuotaQuery.cpp | 4 +- src/Parsers/ASTCreateQuotaQuery.h | 4 +- src/Parsers/ASTCreateRowPolicyQuery.cpp | 4 +- src/Parsers/ASTCreateRowPolicyQuery.h | 4 +- src/Parsers/ASTCreateSettingsProfileQuery.cpp | 4 +- src/Parsers/ASTCreateSettingsProfileQuery.h | 4 +- src/Parsers/ASTCreateUserQuery.cpp | 4 +- src/Parsers/ASTCreateUserQuery.h | 4 +- src/Parsers/ASTGrantQuery.cpp | 4 +- src/Parsers/ASTGrantQuery.h | 6 +- ...ndedRoleSet.cpp => ASTRolesOrUsersSet.cpp} | 6 +- ...ExtendedRoleSet.h => ASTRolesOrUsersSet.h} | 12 ++-- src/Parsers/ASTSetRoleQuery.cpp | 2 +- src/Parsers/ASTSetRoleQuery.h | 6 +- src/Parsers/ParserCreateQuotaQuery.cpp | 14 ++-- src/Parsers/ParserCreateQuotaQuery.h | 2 +- src/Parsers/ParserCreateRoleQuery.cpp | 4 +- src/Parsers/ParserCreateRoleQuery.h | 2 +- src/Parsers/ParserCreateRowPolicyQuery.cpp | 18 +++-- src/Parsers/ParserCreateRowPolicyQuery.h | 2 +- .../ParserCreateSettingsProfileQuery.cpp | 22 +++--- .../ParserCreateSettingsProfileQuery.h | 2 +- src/Parsers/ParserCreateUserQuery.cpp | 20 +++--- src/Parsers/ParserCreateUserQuery.h | 2 +- src/Parsers/ParserExtendedRoleSet.h | 28 -------- src/Parsers/ParserGrantQuery.cpp | 24 ++++--- src/Parsers/ParserGrantQuery.h | 2 +- ...dRoleSet.cpp => ParserRolesOrUsersSet.cpp} | 26 +++---- src/Parsers/ParserRolesOrUsersSet.h | 32 +++++++++ src/Parsers/ParserSetRoleQuery.cpp | 28 ++++---- src/Parsers/ParserSettingsProfileElement.cpp | 8 ++- src/Parsers/ParserSettingsProfileElement.h | 12 ++-- src/Parsers/ya.make | 4 +- .../System/StorageSystemPrivileges.cpp | 2 +- src/Storages/System/StorageSystemQuotas.cpp | 4 +- .../System/StorageSystemRoleGrants.cpp | 4 +- .../System/StorageSystemRowPolicies.cpp | 4 +- .../System/StorageSystemSettingsProfiles.cpp | 4 +- src/Storages/System/StorageSystemUsers.cpp | 4 +- 58 files changed, 308 insertions(+), 271 deletions(-) rename src/Access/{ExtendedRoleSet.cpp => RolesOrUsersSet.cpp} (76%) rename src/Access/{ExtendedRoleSet.h => RolesOrUsersSet.h} (57%) rename src/Parsers/{ASTExtendedRoleSet.cpp => ASTRolesOrUsersSet.cpp} (93%) rename src/Parsers/{ASTExtendedRoleSet.h => ASTRolesOrUsersSet.h} (57%) delete mode 100644 src/Parsers/ParserExtendedRoleSet.h rename src/Parsers/{ParserExtendedRoleSet.cpp => ParserRolesOrUsersSet.cpp} (81%) create mode 100644 src/Parsers/ParserRolesOrUsersSet.h diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 1195bcf842c..4dc91cd8937 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -64,19 +64,23 @@ namespace bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override { - if (ParserCreateUserQuery{}.enableAttachMode(true).parse(pos, node, expected)) - return true; - if (ParserCreateRoleQuery{}.enableAttachMode(true).parse(pos, node, expected)) - return true; - if (ParserCreateRowPolicyQuery{}.enableAttachMode(true).parse(pos, node, expected)) - return true; - if (ParserCreateQuotaQuery{}.enableAttachMode(true).parse(pos, node, expected)) - return true; - if (ParserCreateSettingsProfileQuery{}.enableAttachMode(true).parse(pos, node, expected)) - return true; - if (ParserGrantQuery{}.enableAttachMode(true).parse(pos, node, expected)) - return true; - return false; + ParserCreateUserQuery create_user_p; + ParserCreateRoleQuery create_role_p; + ParserCreateRowPolicyQuery create_policy_p; + ParserCreateQuotaQuery create_quota_p; + ParserCreateSettingsProfileQuery create_profile_p; + ParserGrantQuery grant_p; + + create_user_p.useAttachMode(); + create_role_p.useAttachMode(); + create_policy_p.useAttachMode(); + create_quota_p.useAttachMode(); + create_profile_p.useAttachMode(); + grant_p.useAttachMode(); + + return create_user_p.parse(pos, node, expected) || create_role_p.parse(pos, node, expected) + || create_policy_p.parse(pos, node, expected) || create_quota_p.parse(pos, node, expected) + || create_profile_p.parse(pos, node, expected) || grant_p.parse(pos, node, expected); } }; diff --git a/src/Access/Quota.h b/src/Access/Quota.h index 25b56756dc1..101263e76a5 100644 --- a/src/Access/Quota.h +++ b/src/Access/Quota.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include @@ -91,7 +91,7 @@ struct Quota : public IAccessEntity KeyType key_type = KeyType::NONE; /// Which roles or users should use this quota. - ExtendedRoleSet to_roles; + RolesOrUsersSet to_roles; bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } diff --git a/src/Access/QuotaCache.h b/src/Access/QuotaCache.h index 6e794f0bbd2..0bb5c11a82b 100644 --- a/src/Access/QuotaCache.h +++ b/src/Access/QuotaCache.h @@ -39,7 +39,7 @@ private: QuotaPtr quota; UUID quota_id; - const ExtendedRoleSet * roles = nullptr; + const RolesOrUsersSet * roles = nullptr; std::unordered_map> key_to_intervals; }; diff --git a/src/Access/ExtendedRoleSet.cpp b/src/Access/RolesOrUsersSet.cpp similarity index 76% rename from src/Access/ExtendedRoleSet.cpp rename to src/Access/RolesOrUsersSet.cpp index a8e674b3722..cb0beb42700 100644 --- a/src/Access/ExtendedRoleSet.cpp +++ b/src/Access/RolesOrUsersSet.cpp @@ -1,9 +1,8 @@ - -#include +#include #include #include #include -#include +#include #include #include #include @@ -20,51 +19,51 @@ namespace ErrorCodes } -ExtendedRoleSet::ExtendedRoleSet() = default; -ExtendedRoleSet::ExtendedRoleSet(const ExtendedRoleSet & src) = default; -ExtendedRoleSet & ExtendedRoleSet::operator =(const ExtendedRoleSet & src) = default; -ExtendedRoleSet::ExtendedRoleSet(ExtendedRoleSet && src) = default; -ExtendedRoleSet & ExtendedRoleSet::operator =(ExtendedRoleSet && src) = default; +RolesOrUsersSet::RolesOrUsersSet() = default; +RolesOrUsersSet::RolesOrUsersSet(const RolesOrUsersSet & src) = default; +RolesOrUsersSet & RolesOrUsersSet::operator =(const RolesOrUsersSet & src) = default; +RolesOrUsersSet::RolesOrUsersSet(RolesOrUsersSet && src) = default; +RolesOrUsersSet & RolesOrUsersSet::operator =(RolesOrUsersSet && src) = default; -ExtendedRoleSet::ExtendedRoleSet(AllTag) +RolesOrUsersSet::RolesOrUsersSet(AllTag) { all = true; } -ExtendedRoleSet::ExtendedRoleSet(const UUID & id) +RolesOrUsersSet::RolesOrUsersSet(const UUID & id) { add(id); } -ExtendedRoleSet::ExtendedRoleSet(const std::vector & ids_) +RolesOrUsersSet::RolesOrUsersSet(const std::vector & ids_) { add(ids_); } -ExtendedRoleSet::ExtendedRoleSet(const ASTExtendedRoleSet & ast) +RolesOrUsersSet::RolesOrUsersSet(const ASTRolesOrUsersSet & ast) { init(ast, nullptr); } -ExtendedRoleSet::ExtendedRoleSet(const ASTExtendedRoleSet & ast, const std::optional & current_user_id) +RolesOrUsersSet::RolesOrUsersSet(const ASTRolesOrUsersSet & ast, const std::optional & current_user_id) { init(ast, nullptr, current_user_id); } -ExtendedRoleSet::ExtendedRoleSet(const ASTExtendedRoleSet & ast, const AccessControlManager & manager) +RolesOrUsersSet::RolesOrUsersSet(const ASTRolesOrUsersSet & ast, const AccessControlManager & manager) { init(ast, &manager); } -ExtendedRoleSet::ExtendedRoleSet(const ASTExtendedRoleSet & ast, const AccessControlManager & manager, const std::optional & current_user_id) +RolesOrUsersSet::RolesOrUsersSet(const ASTRolesOrUsersSet & ast, const AccessControlManager & manager, const std::optional & current_user_id) { init(ast, &manager, current_user_id); } -void ExtendedRoleSet::init(const ASTExtendedRoleSet & ast, const AccessControlManager * manager, const std::optional & current_user_id) +void RolesOrUsersSet::init(const ASTRolesOrUsersSet & ast, const AccessControlManager * manager, const std::optional & current_user_id) { all = ast.all; @@ -73,20 +72,20 @@ void ExtendedRoleSet::init(const ASTExtendedRoleSet & ast, const AccessControlMa if (ast.id_mode) return parse(name); assert(manager); - if (ast.can_contain_users && ast.can_contain_roles) + if (ast.allow_user_names && ast.allow_role_names) { auto id = manager->find(name); if (id) return *id; return manager->getID(name); } - else if (ast.can_contain_users) + else if (ast.allow_user_names) { return manager->getID(name); } else { - assert(ast.can_contain_roles); + assert(ast.allow_role_names); return manager->getID(name); } }; @@ -122,9 +121,9 @@ void ExtendedRoleSet::init(const ASTExtendedRoleSet & ast, const AccessControlMa } -std::shared_ptr ExtendedRoleSet::toAST() const +std::shared_ptr RolesOrUsersSet::toAST() const { - auto ast = std::make_shared(); + auto ast = std::make_shared(); ast->id_mode = true; ast->all = all; @@ -148,9 +147,9 @@ std::shared_ptr ExtendedRoleSet::toAST() const } -std::shared_ptr ExtendedRoleSet::toASTWithNames(const AccessControlManager & manager) const +std::shared_ptr RolesOrUsersSet::toASTWithNames(const AccessControlManager & manager) const { - auto ast = std::make_shared(); + auto ast = std::make_shared(); ast->all = all; if (!ids.empty()) @@ -181,21 +180,21 @@ std::shared_ptr ExtendedRoleSet::toASTWithNames(const Access } -String ExtendedRoleSet::toString() const +String RolesOrUsersSet::toString() const { auto ast = toAST(); return serializeAST(*ast); } -String ExtendedRoleSet::toStringWithNames(const AccessControlManager & manager) const +String RolesOrUsersSet::toStringWithNames(const AccessControlManager & manager) const { auto ast = toASTWithNames(manager); return serializeAST(*ast); } -Strings ExtendedRoleSet::toStringsWithNames(const AccessControlManager & manager) const +Strings RolesOrUsersSet::toStringsWithNames(const AccessControlManager & manager) const { if (!all && ids.empty()) return {}; @@ -233,13 +232,13 @@ Strings ExtendedRoleSet::toStringsWithNames(const AccessControlManager & manager } -bool ExtendedRoleSet::empty() const +bool RolesOrUsersSet::empty() const { return ids.empty() && !all; } -void ExtendedRoleSet::clear() +void RolesOrUsersSet::clear() { ids.clear(); all = false; @@ -247,26 +246,26 @@ void ExtendedRoleSet::clear() } -void ExtendedRoleSet::add(const UUID & id) +void RolesOrUsersSet::add(const UUID & id) { ids.insert(id); } -void ExtendedRoleSet::add(const std::vector & ids_) +void RolesOrUsersSet::add(const std::vector & ids_) { for (const auto & id : ids_) add(id); } -bool ExtendedRoleSet::match(const UUID & id) const +bool RolesOrUsersSet::match(const UUID & id) const { return (all || ids.count(id)) && !except_ids.count(id); } -bool ExtendedRoleSet::match(const UUID & user_id, const boost::container::flat_set & enabled_roles) const +bool RolesOrUsersSet::match(const UUID & user_id, const boost::container::flat_set & enabled_roles) const { if (!all && !ids.count(user_id)) { @@ -285,7 +284,7 @@ bool ExtendedRoleSet::match(const UUID & user_id, const boost::container::flat_s } -std::vector ExtendedRoleSet::getMatchingIDs() const +std::vector RolesOrUsersSet::getMatchingIDs() const { if (all) throw Exception("getAllMatchingIDs() can't get ALL ids without manager", ErrorCodes::LOGICAL_ERROR); @@ -295,7 +294,7 @@ std::vector ExtendedRoleSet::getMatchingIDs() const } -std::vector ExtendedRoleSet::getMatchingIDs(const AccessControlManager & manager) const +std::vector RolesOrUsersSet::getMatchingIDs(const AccessControlManager & manager) const { if (!all) return getMatchingIDs(); @@ -316,7 +315,7 @@ std::vector ExtendedRoleSet::getMatchingIDs(const AccessControlManager & m } -bool operator ==(const ExtendedRoleSet & lhs, const ExtendedRoleSet & rhs) +bool operator ==(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs) { return (lhs.all == rhs.all) && (lhs.ids == rhs.ids) && (lhs.except_ids == rhs.except_ids); } diff --git a/src/Access/ExtendedRoleSet.h b/src/Access/RolesOrUsersSet.h similarity index 57% rename from src/Access/ExtendedRoleSet.h rename to src/Access/RolesOrUsersSet.h index eeb4af84f78..bae7f52a574 100644 --- a/src/Access/ExtendedRoleSet.h +++ b/src/Access/RolesOrUsersSet.h @@ -8,35 +8,35 @@ namespace DB { -class ASTExtendedRoleSet; +class ASTRolesOrUsersSet; class AccessControlManager; /// Represents a set of users/roles like /// {user_name | role_name | CURRENT_USER} [,...] | NONE | ALL | ALL EXCEPT {user_name | role_name | CURRENT_USER} [,...] -/// Similar to ASTExtendedRoleSet, but with IDs instead of names. -struct ExtendedRoleSet +/// Similar to ASTRolesOrUsersSet, but with IDs instead of names. +struct RolesOrUsersSet { - ExtendedRoleSet(); - ExtendedRoleSet(const ExtendedRoleSet & src); - ExtendedRoleSet & operator =(const ExtendedRoleSet & src); - ExtendedRoleSet(ExtendedRoleSet && src); - ExtendedRoleSet & operator =(ExtendedRoleSet && src); + RolesOrUsersSet(); + RolesOrUsersSet(const RolesOrUsersSet & src); + RolesOrUsersSet & operator =(const RolesOrUsersSet & src); + RolesOrUsersSet(RolesOrUsersSet && src); + RolesOrUsersSet & operator =(RolesOrUsersSet && src); struct AllTag {}; - ExtendedRoleSet(AllTag); + RolesOrUsersSet(AllTag); - ExtendedRoleSet(const UUID & id); - ExtendedRoleSet(const std::vector & ids_); + RolesOrUsersSet(const UUID & id); + RolesOrUsersSet(const std::vector & ids_); /// The constructor from AST requires the AccessControlManager if `ast.id_mode == false`. - ExtendedRoleSet(const ASTExtendedRoleSet & ast); - ExtendedRoleSet(const ASTExtendedRoleSet & ast, const std::optional & current_user_id); - ExtendedRoleSet(const ASTExtendedRoleSet & ast, const AccessControlManager & manager); - ExtendedRoleSet(const ASTExtendedRoleSet & ast, const AccessControlManager & manager, const std::optional & current_user_id); + RolesOrUsersSet(const ASTRolesOrUsersSet & ast); + RolesOrUsersSet(const ASTRolesOrUsersSet & ast, const std::optional & current_user_id); + RolesOrUsersSet(const ASTRolesOrUsersSet & ast, const AccessControlManager & manager); + RolesOrUsersSet(const ASTRolesOrUsersSet & ast, const AccessControlManager & manager, const std::optional & current_user_id); - std::shared_ptr toAST() const; - std::shared_ptr toASTWithNames(const AccessControlManager & manager) const; + std::shared_ptr toAST() const; + std::shared_ptr toASTWithNames(const AccessControlManager & manager) const; String toString() const; String toStringWithNames(const AccessControlManager & manager) const; @@ -47,7 +47,7 @@ struct ExtendedRoleSet void add(const UUID & id); void add(const std::vector & ids_); - /// Checks if a specified ID matches this ExtendedRoleSet. + /// Checks if a specified ID matches this RolesOrUsersSet. bool match(const UUID & id) const; bool match(const UUID & user_id, const boost::container::flat_set & enabled_roles) const; @@ -57,15 +57,15 @@ struct ExtendedRoleSet /// Returns a list of matching users and roles. std::vector getMatchingIDs(const AccessControlManager & manager) const; - friend bool operator ==(const ExtendedRoleSet & lhs, const ExtendedRoleSet & rhs); - friend bool operator !=(const ExtendedRoleSet & lhs, const ExtendedRoleSet & rhs) { return !(lhs == rhs); } + friend bool operator ==(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs); + friend bool operator !=(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs) { return !(lhs == rhs); } boost::container::flat_set ids; bool all = false; boost::container::flat_set except_ids; private: - void init(const ASTExtendedRoleSet & ast, const AccessControlManager * manager = nullptr, const std::optional & current_user_id = {}); + void init(const ASTRolesOrUsersSet & ast, const AccessControlManager * manager = nullptr, const std::optional & current_user_id = {}); }; } diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h index 9d5b00b427d..c9b4d69152d 100644 --- a/src/Access/RowPolicy.h +++ b/src/Access/RowPolicy.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include @@ -91,7 +91,7 @@ struct RowPolicy : public IAccessEntity Type getType() const override { return TYPE; } /// Which roles or users should use this row policy. - ExtendedRoleSet to_roles; + RolesOrUsersSet to_roles; private: void setName(const String & name_) override; diff --git a/src/Access/RowPolicyCache.h b/src/Access/RowPolicyCache.h index 139949ae815..f7270c6fce9 100644 --- a/src/Access/RowPolicyCache.h +++ b/src/Access/RowPolicyCache.h @@ -27,7 +27,7 @@ private: void setPolicy(const RowPolicyPtr & policy_); RowPolicyPtr policy; - const ExtendedRoleSet * roles = nullptr; + const RolesOrUsersSet * roles = nullptr; std::shared_ptr> database_and_table_name; ASTPtr parsed_conditions[RowPolicy::MAX_CONDITION_TYPE]; }; diff --git a/src/Access/SettingsProfile.h b/src/Access/SettingsProfile.h index 9589b5b3eb5..210aa47c358 100644 --- a/src/Access/SettingsProfile.h +++ b/src/Access/SettingsProfile.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include @@ -14,7 +14,7 @@ struct SettingsProfile : public IAccessEntity SettingsProfileElements elements; /// Which roles or users should use this settings profile. - ExtendedRoleSet to_roles; + RolesOrUsersSet to_roles; bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } diff --git a/src/Access/User.h b/src/Access/User.h index da2fb14e131..4852fce375d 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include @@ -19,7 +19,7 @@ struct User : public IAccessEntity AllowedClientHosts allowed_client_hosts = AllowedClientHosts::AnyHostTag{}; GrantedAccess access; GrantedRoles granted_roles; - ExtendedRoleSet default_roles = ExtendedRoleSet::AllTag{}; + RolesOrUsersSet default_roles = RolesOrUsersSet::AllTag{}; SettingsProfileElements settings; bool equal(const IAccessEntity & other) const override; diff --git a/src/Access/ya.make b/src/Access/ya.make index 970c0714a93..bdd62ae2b7b 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -17,7 +17,6 @@ SRCS( EnabledRolesInfo.cpp EnabledRowPolicies.cpp EnabledSettings.cpp - ExtendedRoleSet.cpp GrantedAccess.cpp GrantedRoles.cpp IAccessEntity.cpp @@ -29,6 +28,7 @@ SRCS( QuotaUsage.cpp Role.cpp RoleCache.cpp + RolesOrUsersSet.cpp RowPolicy.cpp RowPolicyCache.cpp SettingsConstraints.cpp diff --git a/src/Interpreters/InterpreterCreateQuotaQuery.cpp b/src/Interpreters/InterpreterCreateQuotaQuery.cpp index 0cca163beec..f45c2c9709d 100644 --- a/src/Interpreters/InterpreterCreateQuotaQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuotaQuery.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -19,7 +19,7 @@ namespace Quota & quota, const ASTCreateQuotaQuery & query, const String & override_name, - const std::optional & override_to_roles) + const std::optional & override_to_roles) { if (!override_name.empty()) quota.setName(override_name); @@ -82,9 +82,9 @@ BlockIO InterpreterCreateQuotaQuery::execute() return executeDDLQueryOnCluster(query_ptr, context); } - std::optional roles_from_query; + std::optional roles_from_query; if (query.roles) - roles_from_query = ExtendedRoleSet{*query.roles, access_control, context.getUserID()}; + roles_from_query = RolesOrUsersSet{*query.roles, access_control, context.getUserID()}; if (query.alter) { diff --git a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp index bfd7d60b397..3a0ee3f16a1 100644 --- a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp +++ b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include #include @@ -18,7 +18,7 @@ namespace RowPolicy & policy, const ASTCreateRowPolicyQuery & query, const RowPolicy::NameParts & override_name, - const std::optional & override_to_roles) + const std::optional & override_to_roles) { if (!override_name.empty()) policy.setNameParts(override_name); @@ -58,9 +58,9 @@ BlockIO InterpreterCreateRowPolicyQuery::execute() } assert(query.names->cluster.empty()); - std::optional roles_from_query; + std::optional roles_from_query; if (query.roles) - roles_from_query = ExtendedRoleSet{*query.roles, access_control, context.getUserID()}; + roles_from_query = RolesOrUsersSet{*query.roles, access_control, context.getUserID()}; query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); diff --git a/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp b/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp index ac2a4249986..2d5f4d499b7 100644 --- a/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp +++ b/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -17,7 +17,7 @@ namespace const ASTCreateSettingsProfileQuery & query, const String & override_name, const std::optional & override_settings, - const std::optional & override_to_roles) + const std::optional & override_to_roles) { if (!override_name.empty()) profile.setName(override_name); @@ -58,9 +58,9 @@ BlockIO InterpreterCreateSettingsProfileQuery::execute() if (query.settings) settings_from_query = SettingsProfileElements{*query.settings, access_control}; - std::optional roles_from_query; + std::optional roles_from_query; if (query.to_roles) - roles_from_query = ExtendedRoleSet{*query.to_roles, access_control, context.getUserID()}; + roles_from_query = RolesOrUsersSet{*query.to_roles, access_control, context.getUserID()}; if (query.alter) { diff --git a/src/Interpreters/InterpreterCreateUserQuery.cpp b/src/Interpreters/InterpreterCreateUserQuery.cpp index 8b57703f08c..111f698beb9 100644 --- a/src/Interpreters/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/InterpreterCreateUserQuery.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -19,7 +19,7 @@ namespace User & user, const ASTCreateUserQuery & query, const std::shared_ptr & override_name, - const std::optional & override_default_roles, + const std::optional & override_default_roles, const std::optional & override_settings) { if (override_name) @@ -45,7 +45,7 @@ namespace if (query.add_hosts) user.allowed_client_hosts.add(*query.add_hosts); - auto set_default_roles = [&](const ExtendedRoleSet & default_roles_) + auto set_default_roles = [&](const RolesOrUsersSet & default_roles_) { if (!query.alter && !default_roles_.all) user.granted_roles.grant(default_roles_.getMatchingIDs()); @@ -73,10 +73,10 @@ BlockIO InterpreterCreateUserQuery::execute() auto access = context.getAccess(); access->checkAccess(query.alter ? AccessType::ALTER_USER : AccessType::CREATE_USER); - std::optional default_roles_from_query; + std::optional default_roles_from_query; if (query.default_roles) { - default_roles_from_query = ExtendedRoleSet{*query.default_roles, access_control}; + default_roles_from_query = RolesOrUsersSet{*query.default_roles, access_control}; if (!query.alter && !default_roles_from_query->all) { for (const UUID & role : default_roles_from_query->getMatchingIDs()) diff --git a/src/Interpreters/InterpreterGrantQuery.cpp b/src/Interpreters/InterpreterGrantQuery.cpp index c72e48c2019..8981c06f962 100644 --- a/src/Interpreters/InterpreterGrantQuery.cpp +++ b/src/Interpreters/InterpreterGrantQuery.cpp @@ -1,11 +1,11 @@ #include #include -#include +#include #include #include #include #include -#include +#include #include #include #include @@ -74,7 +74,7 @@ BlockIO InterpreterGrantQuery::execute() std::vector roles_from_query; if (query.roles) { - roles_from_query = ExtendedRoleSet{*query.roles, access_control}.getMatchingIDs(access_control); + roles_from_query = RolesOrUsersSet{*query.roles, access_control}.getMatchingIDs(access_control); for (const UUID & role_from_query : roles_from_query) access->checkAdminOption(role_from_query); } @@ -85,7 +85,7 @@ BlockIO InterpreterGrantQuery::execute() return executeDDLQueryOnCluster(query_ptr, context); } - std::vector to_roles = ExtendedRoleSet{*query.to_roles, access_control, context.getUserID()}.getMatchingIDs(access_control); + std::vector to_roles = RolesOrUsersSet{*query.to_roles, access_control, context.getUserID()}.getMatchingIDs(access_control); String current_database = context.getCurrentDatabase(); auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr @@ -115,7 +115,7 @@ void InterpreterGrantQuery::updateUserFromQuery(User & user, const ASTGrantQuery { std::vector roles_from_query; if (query.roles) - roles_from_query = ExtendedRoleSet{*query.roles}.getMatchingIDs(); + roles_from_query = RolesOrUsersSet{*query.roles}.getMatchingIDs(); updateFromQueryImpl(user, query, roles_from_query, {}); } @@ -124,7 +124,7 @@ void InterpreterGrantQuery::updateRoleFromQuery(Role & role, const ASTGrantQuery { std::vector roles_from_query; if (query.roles) - roles_from_query = ExtendedRoleSet{*query.roles}.getMatchingIDs(); + roles_from_query = RolesOrUsersSet{*query.roles}.getMatchingIDs(); updateFromQueryImpl(role, query, roles_from_query, {}); } diff --git a/src/Interpreters/InterpreterSetRoleQuery.cpp b/src/Interpreters/InterpreterSetRoleQuery.cpp index c627061dd51..f955c881b2e 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.cpp +++ b/src/Interpreters/InterpreterSetRoleQuery.cpp @@ -1,8 +1,8 @@ #include #include -#include +#include #include -#include +#include #include #include @@ -38,7 +38,7 @@ void InterpreterSetRoleQuery::setRole(const ASTSetRoleQuery & query) } else { - ExtendedRoleSet roles_from_query{*query.roles, access_control}; + RolesOrUsersSet roles_from_query{*query.roles, access_control}; boost::container::flat_set new_current_roles; if (roles_from_query.all) { @@ -65,8 +65,8 @@ void InterpreterSetRoleQuery::setDefaultRole(const ASTSetRoleQuery & query) context.checkAccess(AccessType::ALTER_USER); auto & access_control = context.getAccessControlManager(); - std::vector to_users = ExtendedRoleSet{*query.to_users, access_control, context.getUserID()}.getMatchingIDs(access_control); - ExtendedRoleSet roles_from_query{*query.roles, access_control}; + std::vector to_users = RolesOrUsersSet{*query.to_users, access_control, context.getUserID()}.getMatchingIDs(access_control); + RolesOrUsersSet roles_from_query{*query.roles, access_control}; auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { @@ -79,7 +79,7 @@ void InterpreterSetRoleQuery::setDefaultRole(const ASTSetRoleQuery & query) } -void InterpreterSetRoleQuery::updateUserSetDefaultRoles(User & user, const ExtendedRoleSet & roles_from_query) +void InterpreterSetRoleQuery::updateUserSetDefaultRoles(User & user, const RolesOrUsersSet & roles_from_query) { if (!roles_from_query.all) { diff --git a/src/Interpreters/InterpreterSetRoleQuery.h b/src/Interpreters/InterpreterSetRoleQuery.h index 91cf5fc1b2e..0919b0f23f9 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.h +++ b/src/Interpreters/InterpreterSetRoleQuery.h @@ -9,7 +9,7 @@ namespace DB class Context; class ASTSetRoleQuery; -struct ExtendedRoleSet; +struct RolesOrUsersSet; struct User; @@ -20,7 +20,7 @@ public: BlockIO execute() override; - static void updateUserSetDefaultRoles(User & user, const ExtendedRoleSet & roles_from_query); + static void updateUserSetDefaultRoles(User & user, const RolesOrUsersSet & roles_from_query); private: void setRole(const ASTSetRoleQuery & query); diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index d3ab5ac5001..9c28c3d0bd2 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include @@ -51,7 +51,7 @@ namespace if (user.allowed_client_hosts != AllowedClientHosts::AnyHostTag{}) query->hosts = user.allowed_client_hosts; - if (user.default_roles != ExtendedRoleSet::AllTag{}) + if (user.default_roles != RolesOrUsersSet::AllTag{}) { if (attach_mode) query->default_roles = user.default_roles.toAST(); diff --git a/src/Interpreters/InterpreterShowGrantsQuery.cpp b/src/Interpreters/InterpreterShowGrantsQuery.cpp index 130749526c7..c6e3ccce7c7 100644 --- a/src/Interpreters/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/InterpreterShowGrantsQuery.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include #include @@ -29,7 +29,7 @@ namespace { ASTs res; - std::shared_ptr to_roles = std::make_shared(); + std::shared_ptr to_roles = std::make_shared(); to_roles->names.push_back(grantee.getName()); auto grants_and_partial_revokes = grantee.access.getGrantsAndPartialRevokes(); @@ -87,9 +87,9 @@ namespace grant_query->admin_option = admin_option; grant_query->to_roles = to_roles; if (attach_mode) - grant_query->roles = ExtendedRoleSet{roles}.toAST(); + grant_query->roles = RolesOrUsersSet{roles}.toAST(); else - grant_query->roles = ExtendedRoleSet{roles}.toASTWithNames(*manager); + grant_query->roles = RolesOrUsersSet{roles}.toASTWithNames(*manager); res.push_back(std::move(grant_query)); } diff --git a/src/Parsers/ASTCreateQuotaQuery.cpp b/src/Parsers/ASTCreateQuotaQuery.cpp index d33af6126f1..fc4e2edb9e7 100644 --- a/src/Parsers/ASTCreateQuotaQuery.cpp +++ b/src/Parsers/ASTCreateQuotaQuery.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -104,7 +104,7 @@ namespace } } - void formatToRoles(const ASTExtendedRoleSet & roles, const IAST::FormatSettings & settings) + void formatToRoles(const ASTRolesOrUsersSet & roles, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); roles.format(settings); diff --git a/src/Parsers/ASTCreateQuotaQuery.h b/src/Parsers/ASTCreateQuotaQuery.h index 370083f4e25..002c374322f 100644 --- a/src/Parsers/ASTCreateQuotaQuery.h +++ b/src/Parsers/ASTCreateQuotaQuery.h @@ -7,7 +7,7 @@ namespace DB { -class ASTExtendedRoleSet; +class ASTRolesOrUsersSet; /** CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name @@ -51,7 +51,7 @@ public: }; std::vector all_limits; - std::shared_ptr roles; + std::shared_ptr roles; String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ASTCreateRowPolicyQuery.cpp b/src/Parsers/ASTCreateRowPolicyQuery.cpp index caa52e3ac58..580642f2da5 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/ASTCreateRowPolicyQuery.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -116,7 +116,7 @@ namespace } - void formatToRoles(const ASTExtendedRoleSet & roles, const IAST::FormatSettings & settings) + void formatToRoles(const ASTRolesOrUsersSet & roles, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); roles.format(settings); diff --git a/src/Parsers/ASTCreateRowPolicyQuery.h b/src/Parsers/ASTCreateRowPolicyQuery.h index af561b47e12..4a7572eaefd 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.h +++ b/src/Parsers/ASTCreateRowPolicyQuery.h @@ -10,7 +10,7 @@ namespace DB { class ASTRowPolicyNames; -class ASTExtendedRoleSet; +class ASTRolesOrUsersSet; /** CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] name ON [database.]table * [AS {PERMISSIVE | RESTRICTIVE}] @@ -43,7 +43,7 @@ public: std::optional is_restrictive; std::array, RowPolicy::MAX_CONDITION_TYPE> conditions; /// `nullopt` means "not set", `nullptr` means set to NONE. - std::shared_ptr roles; + std::shared_ptr roles; String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ASTCreateSettingsProfileQuery.cpp b/src/Parsers/ASTCreateSettingsProfileQuery.cpp index 21d8c20ffc1..77c2f1b22d7 100644 --- a/src/Parsers/ASTCreateSettingsProfileQuery.cpp +++ b/src/Parsers/ASTCreateSettingsProfileQuery.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include @@ -32,7 +32,7 @@ namespace settings.format(format); } - void formatToRoles(const ASTExtendedRoleSet & roles, const IAST::FormatSettings & settings) + void formatToRoles(const ASTRolesOrUsersSet & roles, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); roles.format(settings); diff --git a/src/Parsers/ASTCreateSettingsProfileQuery.h b/src/Parsers/ASTCreateSettingsProfileQuery.h index bb2a9474504..119019093b2 100644 --- a/src/Parsers/ASTCreateSettingsProfileQuery.h +++ b/src/Parsers/ASTCreateSettingsProfileQuery.h @@ -7,7 +7,7 @@ namespace DB { class ASTSettingsProfileElements; -class ASTExtendedRoleSet; +class ASTRolesOrUsersSet; /** CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name @@ -34,7 +34,7 @@ public: std::shared_ptr settings; - std::shared_ptr to_roles; + std::shared_ptr to_roles; String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ASTCreateUserQuery.cpp b/src/Parsers/ASTCreateUserQuery.cpp index 60f61fbf51f..2ba454b3d65 100644 --- a/src/Parsers/ASTCreateUserQuery.cpp +++ b/src/Parsers/ASTCreateUserQuery.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include @@ -167,7 +167,7 @@ namespace } - void formatDefaultRoles(const ASTExtendedRoleSet & default_roles, const IAST::FormatSettings & settings) + void formatDefaultRoles(const ASTRolesOrUsersSet & default_roles, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " DEFAULT ROLE " << (settings.hilite ? IAST::hilite_none : ""); default_roles.format(settings); diff --git a/src/Parsers/ASTCreateUserQuery.h b/src/Parsers/ASTCreateUserQuery.h index 565c82bc98e..5c8f8fcf563 100644 --- a/src/Parsers/ASTCreateUserQuery.h +++ b/src/Parsers/ASTCreateUserQuery.h @@ -9,7 +9,7 @@ namespace DB { class ASTUserNamesWithHost; -class ASTExtendedRoleSet; +class ASTRolesOrUsersSet; class ASTSettingsProfileElements; /** CREATE USER [IF NOT EXISTS | OR REPLACE] name @@ -45,7 +45,7 @@ public: std::optional add_hosts; std::optional remove_hosts; - std::shared_ptr default_roles; + std::shared_ptr default_roles; std::shared_ptr settings; diff --git a/src/Parsers/ASTGrantQuery.cpp b/src/Parsers/ASTGrantQuery.cpp index 8114bef0766..cf1943477b2 100644 --- a/src/Parsers/ASTGrantQuery.cpp +++ b/src/Parsers/ASTGrantQuery.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include @@ -75,7 +75,7 @@ namespace } - void formatToRoles(const ASTExtendedRoleSet & to_roles, ASTGrantQuery::Kind kind, const IAST::FormatSettings & settings) + void formatToRoles(const ASTRolesOrUsersSet & to_roles, ASTGrantQuery::Kind kind, const IAST::FormatSettings & settings) { using Kind = ASTGrantQuery::Kind; settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << ((kind == Kind::GRANT) ? " TO " : " FROM ") diff --git a/src/Parsers/ASTGrantQuery.h b/src/Parsers/ASTGrantQuery.h index 7e3321799fb..9a11f5dc509 100644 --- a/src/Parsers/ASTGrantQuery.h +++ b/src/Parsers/ASTGrantQuery.h @@ -7,7 +7,7 @@ namespace DB { -class ASTExtendedRoleSet; +class ASTRolesOrUsersSet; /** GRANT access_type[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*} TO {user_name | CURRENT_USER} [,...] [WITH GRANT OPTION] @@ -27,8 +27,8 @@ public: Kind kind = Kind::GRANT; bool attach = false; AccessRightsElements access_rights_elements; - std::shared_ptr roles; - std::shared_ptr to_roles; + std::shared_ptr roles; + std::shared_ptr to_roles; bool grant_option = false; bool admin_option = false; diff --git a/src/Parsers/ASTExtendedRoleSet.cpp b/src/Parsers/ASTRolesOrUsersSet.cpp similarity index 93% rename from src/Parsers/ASTExtendedRoleSet.cpp rename to src/Parsers/ASTRolesOrUsersSet.cpp index 1803af11ab3..a666d8ae1d5 100644 --- a/src/Parsers/ASTExtendedRoleSet.cpp +++ b/src/Parsers/ASTRolesOrUsersSet.cpp @@ -1,4 +1,4 @@ -#include +#include #include @@ -20,7 +20,7 @@ namespace } } -void ASTExtendedRoleSet::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +void ASTRolesOrUsersSet::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { if (empty()) { @@ -74,7 +74,7 @@ void ASTExtendedRoleSet::formatImpl(const FormatSettings & settings, FormatState } -void ASTExtendedRoleSet::replaceCurrentUserTagWithName(const String & current_user_name) +void ASTRolesOrUsersSet::replaceCurrentUserTagWithName(const String & current_user_name) { if (current_user) { diff --git a/src/Parsers/ASTExtendedRoleSet.h b/src/Parsers/ASTRolesOrUsersSet.h similarity index 57% rename from src/Parsers/ASTExtendedRoleSet.h rename to src/Parsers/ASTRolesOrUsersSet.h index 656f563bd9a..f257ce1066c 100644 --- a/src/Parsers/ASTExtendedRoleSet.h +++ b/src/Parsers/ASTRolesOrUsersSet.h @@ -7,7 +7,7 @@ namespace DB { /// Represents a set of users/roles like /// {user_name | role_name | CURRENT_USER} [,...] | NONE | ALL | ALL EXCEPT {user_name | role_name | CURRENT_USER} [,...] -class ASTExtendedRoleSet : public IAST +class ASTRolesOrUsersSet : public IAST { public: Strings names; @@ -16,15 +16,15 @@ public: Strings except_names; bool except_current_user = false; - bool id_mode = false; /// true if `names` and `except_names` keep UUIDs, not names. - bool can_contain_roles = true; /// true if this set can contain names of roles. - bool can_contain_users = true; /// true if this set can contain names of users. + bool id_mode = false; /// true if `names` and `except_names` keep UUIDs, not names. + bool allow_role_names = true; /// true if this set can contain names of roles. + bool allow_user_names = true; /// true if this set can contain names of users. bool empty() const { return names.empty() && !current_user && !all; } void replaceCurrentUserTagWithName(const String & current_user_name); - String getID(char) const override { return "ExtendedRoleSet"; } - ASTPtr clone() const override { return std::make_shared(*this); } + String getID(char) const override { return "RolesOrUsersSet"; } + ASTPtr clone() const override { return std::make_shared(*this); } void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; } diff --git a/src/Parsers/ASTSetRoleQuery.cpp b/src/Parsers/ASTSetRoleQuery.cpp index 0c8842fdac6..b5e0c05e083 100644 --- a/src/Parsers/ASTSetRoleQuery.cpp +++ b/src/Parsers/ASTSetRoleQuery.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include diff --git a/src/Parsers/ASTSetRoleQuery.h b/src/Parsers/ASTSetRoleQuery.h index 8f1fb357d86..f0170ae6af2 100644 --- a/src/Parsers/ASTSetRoleQuery.h +++ b/src/Parsers/ASTSetRoleQuery.h @@ -5,7 +5,7 @@ namespace DB { -class ASTExtendedRoleSet; +class ASTRolesOrUsersSet; /** SET ROLE {DEFAULT | NONE | role [,...] | ALL | ALL EXCEPT role [,...]} * SET DEFAULT ROLE {NONE | role [,...] | ALL | ALL EXCEPT role [,...]} TO {user|CURRENT_USER} [,...] @@ -21,8 +21,8 @@ public: }; Kind kind = Kind::SET_ROLE; - std::shared_ptr roles; - std::shared_ptr to_users; + std::shared_ptr roles; + std::shared_ptr to_users; String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ParserCreateQuotaQuery.cpp b/src/Parsers/ParserCreateQuotaQuery.cpp index 5eb138d6be0..e953f698de0 100644 --- a/src/Parsers/ParserCreateQuotaQuery.cpp +++ b/src/Parsers/ParserCreateQuotaQuery.cpp @@ -3,10 +3,10 @@ #include #include #include -#include +#include #include #include -#include +#include #include #include @@ -185,15 +185,17 @@ namespace }); } - bool parseToRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) + bool parseToRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) { return IParserBase::wrapParseImpl(pos, [&] { ASTPtr node; - if (roles || !ParserKeyword{"TO"}.ignore(pos, expected) || !ParserExtendedRoleSet{}.useIDMode(id_mode).parse(pos, node, expected)) + ParserRolesOrUsersSet roles_p; + roles_p.allowAll().allowRoleNames().allowUserNames().allowCurrentUser().useIDMode(id_mode); + if (roles || !ParserKeyword{"TO"}.ignore(pos, expected) || !roles_p.parse(pos, node, expected)) return false; - roles = std::static_pointer_cast(node); + roles = std::static_pointer_cast(node); return true; }); } @@ -266,7 +268,7 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe break; } - std::shared_ptr roles; + std::shared_ptr roles; parseToRoles(pos, expected, attach_mode, roles); if (cluster.empty()) diff --git a/src/Parsers/ParserCreateQuotaQuery.h b/src/Parsers/ParserCreateQuotaQuery.h index 786c8292b15..e2185d4d5ff 100644 --- a/src/Parsers/ParserCreateQuotaQuery.h +++ b/src/Parsers/ParserCreateQuotaQuery.h @@ -24,7 +24,7 @@ namespace DB class ParserCreateQuotaQuery : public IParserBase { public: - ParserCreateQuotaQuery & enableAttachMode(bool enable_) { attach_mode = enable_; return *this; } + void useAttachMode(bool attach_mode_ = true) { attach_mode = attach_mode_; } protected: const char * getName() const override { return "CREATE QUOTA or ALTER QUOTA query"; } diff --git a/src/Parsers/ParserCreateRoleQuery.cpp b/src/Parsers/ParserCreateRoleQuery.cpp index 08dd31c51a3..6feeefa4657 100644 --- a/src/Parsers/ParserCreateRoleQuery.cpp +++ b/src/Parsers/ParserCreateRoleQuery.cpp @@ -31,7 +31,9 @@ namespace return false; ASTPtr new_settings_ast; - if (!ParserSettingsProfileElements{}.useIDMode(id_mode).parse(pos, new_settings_ast, expected)) + ParserSettingsProfileElements elements_p; + elements_p.useIDMode(id_mode); + if (!elements_p.parse(pos, new_settings_ast, expected)) return false; if (!settings) diff --git a/src/Parsers/ParserCreateRoleQuery.h b/src/Parsers/ParserCreateRoleQuery.h index 2afeb7f7ec4..1fdee67eaab 100644 --- a/src/Parsers/ParserCreateRoleQuery.h +++ b/src/Parsers/ParserCreateRoleQuery.h @@ -16,7 +16,7 @@ namespace DB class ParserCreateRoleQuery : public IParserBase { public: - ParserCreateRoleQuery & enableAttachMode(bool enable) { attach_mode = enable; return *this; } + void useAttachMode(bool attach_mode_ = true) { attach_mode = attach_mode_; } protected: const char * getName() const override { return "CREATE ROLE or ALTER ROLE query"; } diff --git a/src/Parsers/ParserCreateRowPolicyQuery.cpp b/src/Parsers/ParserCreateRowPolicyQuery.cpp index 4f5f2989a7b..061cca4ce63 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/ParserCreateRowPolicyQuery.cpp @@ -3,8 +3,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include @@ -173,16 +173,20 @@ namespace }); } - bool parseToRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) + bool parseToRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) { return IParserBase::wrapParseImpl(pos, [&] { ASTPtr ast; - if (roles || !ParserKeyword{"TO"}.ignore(pos, expected) - || !ParserExtendedRoleSet{}.useIDMode(id_mode).parse(pos, ast, expected)) + if (roles || !ParserKeyword{"TO"}.ignore(pos, expected)) return false; - roles = std::static_pointer_cast(ast); + ParserRolesOrUsersSet roles_p; + roles_p.allowAll().allowRoleNames().allowUserNames().allowCurrentUser().useIDMode(id_mode); + if (!roles_p.parse(pos, ast, expected)) + return false; + + roles = std::static_pointer_cast(ast); return true; }); } @@ -259,7 +263,7 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & break; } - std::shared_ptr roles; + std::shared_ptr roles; parseToRoles(pos, expected, attach_mode, roles); if (cluster.empty()) diff --git a/src/Parsers/ParserCreateRowPolicyQuery.h b/src/Parsers/ParserCreateRowPolicyQuery.h index 5110998e53d..1a64f2e893c 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.h +++ b/src/Parsers/ParserCreateRowPolicyQuery.h @@ -24,7 +24,7 @@ namespace DB class ParserCreateRowPolicyQuery : public IParserBase { public: - ParserCreateRowPolicyQuery & enableAttachMode(bool enable_) { attach_mode = enable_; return *this; } + void useAttachMode(bool attach_mode_ = true) { attach_mode = attach_mode_; } protected: const char * getName() const override { return "CREATE ROW POLICY or ALTER ROW POLICY query"; } diff --git a/src/Parsers/ParserCreateSettingsProfileQuery.cpp b/src/Parsers/ParserCreateSettingsProfileQuery.cpp index 4d5a9c09ad8..56bd39b9230 100644 --- a/src/Parsers/ParserCreateSettingsProfileQuery.cpp +++ b/src/Parsers/ParserCreateSettingsProfileQuery.cpp @@ -5,8 +5,8 @@ #include #include #include -#include -#include +#include +#include #include @@ -33,7 +33,9 @@ namespace return false; ASTPtr new_settings_ast; - if (!ParserSettingsProfileElements{}.useIDMode(id_mode).enableInheritKeyword(true).parse(pos, new_settings_ast, expected)) + ParserSettingsProfileElements elements_p; + elements_p.useInheritKeyword(true).useIDMode(id_mode); + if (!elements_p.parse(pos, new_settings_ast, expected)) return false; if (!settings) @@ -44,16 +46,20 @@ namespace }); } - bool parseToRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) + bool parseToRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) { return IParserBase::wrapParseImpl(pos, [&] { ASTPtr ast; - if (roles || !ParserKeyword{"TO"}.ignore(pos, expected) - || !ParserExtendedRoleSet{}.useIDMode(id_mode).parse(pos, ast, expected)) + if (roles || !ParserKeyword{"TO"}.ignore(pos, expected)) return false; - roles = std::static_pointer_cast(ast); + ParserRolesOrUsersSet roles_p; + roles_p.allowAll().allowRoleNames().allowUserNames().allowCurrentUser().useIDMode(id_mode); + if (!roles_p.parse(pos, ast, expected)) + return false; + + roles = std::static_pointer_cast(ast); return true; }); } @@ -122,7 +128,7 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec break; } - std::shared_ptr to_roles; + std::shared_ptr to_roles; parseToRoles(pos, expected, attach_mode, to_roles); if (cluster.empty()) diff --git a/src/Parsers/ParserCreateSettingsProfileQuery.h b/src/Parsers/ParserCreateSettingsProfileQuery.h index 073a8ca75ae..ab730fcd8eb 100644 --- a/src/Parsers/ParserCreateSettingsProfileQuery.h +++ b/src/Parsers/ParserCreateSettingsProfileQuery.h @@ -16,7 +16,7 @@ namespace DB class ParserCreateSettingsProfileQuery : public IParserBase { public: - ParserCreateSettingsProfileQuery & enableAttachMode(bool enable) { attach_mode = enable; return *this; } + void useAttachMode(bool attach_mode_ = true) { attach_mode = attach_mode_; } protected: const char * getName() const override { return "CREATE SETTINGS PROFILE or ALTER SETTINGS PROFILE query"; } diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index e99457a2f87..ff7f2dc8790 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -7,9 +7,9 @@ #include #include #include -#include +#include #include -#include +#include #include #include #include @@ -186,7 +186,7 @@ namespace } - bool parseDefaultRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & default_roles) + bool parseDefaultRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & default_roles) { return IParserBase::wrapParseImpl(pos, [&] { @@ -194,11 +194,13 @@ namespace return false; ASTPtr ast; - if (!ParserExtendedRoleSet{}.enableCurrentUserKeyword(false).useIDMode(id_mode).parse(pos, ast, expected)) + ParserRolesOrUsersSet default_roles_p; + default_roles_p.allowAll().allowRoleNames().useIDMode(id_mode); + if (!default_roles_p.parse(pos, ast, expected)) return false; - default_roles = typeid_cast>(ast); - default_roles->can_contain_users = false; + default_roles = typeid_cast>(ast); + default_roles->allow_user_names = false; return true; }); } @@ -212,7 +214,9 @@ namespace return false; ASTPtr new_settings_ast; - if (!ParserSettingsProfileElements{}.useIDMode(id_mode).parse(pos, new_settings_ast, expected)) + ParserSettingsProfileElements elements_p; + elements_p.useInheritKeyword(true).useIDMode(id_mode); + if (!elements_p.parse(pos, new_settings_ast, expected)) return false; if (!settings) @@ -276,7 +280,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec std::optional hosts; std::optional add_hosts; std::optional remove_hosts; - std::shared_ptr default_roles; + std::shared_ptr default_roles; std::shared_ptr settings; String cluster; diff --git a/src/Parsers/ParserCreateUserQuery.h b/src/Parsers/ParserCreateUserQuery.h index 2a890f41060..1628f5ea5b9 100644 --- a/src/Parsers/ParserCreateUserQuery.h +++ b/src/Parsers/ParserCreateUserQuery.h @@ -20,7 +20,7 @@ namespace DB class ParserCreateUserQuery : public IParserBase { public: - ParserCreateUserQuery & enableAttachMode(bool enable) { attach_mode = enable; return *this; } + ParserCreateUserQuery & useAttachMode(bool attach_mode_ = true) { attach_mode = attach_mode_; return *this; } protected: const char * getName() const override { return "CREATE USER or ALTER USER query"; } diff --git a/src/Parsers/ParserExtendedRoleSet.h b/src/Parsers/ParserExtendedRoleSet.h deleted file mode 100644 index df723786bd9..00000000000 --- a/src/Parsers/ParserExtendedRoleSet.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ -/** Parses a string like this: - * {role|CURRENT_USER} [,...] | NONE | ALL | ALL EXCEPT {role|CURRENT_USER} [,...] - */ -class ParserExtendedRoleSet : public IParserBase -{ -public: - ParserExtendedRoleSet & enableAllKeyword(bool enable_) { all_keyword = enable_; return *this; } - ParserExtendedRoleSet & enableCurrentUserKeyword(bool enable_) { current_user_keyword = enable_; return *this; } - ParserExtendedRoleSet & useIDMode(bool enable_) { id_mode = enable_; return *this; } - -protected: - const char * getName() const override { return "ExtendedRoleSet"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; - -private: - bool all_keyword = true; - bool current_user_keyword = true; - bool id_mode = false; -}; - -} diff --git a/src/Parsers/ParserGrantQuery.cpp b/src/Parsers/ParserGrantQuery.cpp index 64dde8f6524..03c0daa08a3 100644 --- a/src/Parsers/ParserGrantQuery.cpp +++ b/src/Parsers/ParserGrantQuery.cpp @@ -1,10 +1,10 @@ #include #include #include -#include +#include #include #include -#include +#include #include @@ -199,21 +199,23 @@ namespace } - bool parseRoles(IParser::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) + bool parseRoles(IParser::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) { return IParserBase::wrapParseImpl(pos, [&] { ASTPtr ast; - if (!ParserExtendedRoleSet{}.enableAllKeyword(false).enableCurrentUserKeyword(false).useIDMode(id_mode).parse(pos, ast, expected)) + ParserRolesOrUsersSet roles_p; + roles_p.allowRoleNames().useIDMode(id_mode); + if (!roles_p.parse(pos, ast, expected)) return false; - roles = typeid_cast>(ast); + roles = typeid_cast>(ast); return true; }); } - bool parseToRoles(IParser::Pos & pos, Expected & expected, ASTGrantQuery::Kind kind, std::shared_ptr & to_roles) + bool parseToRoles(IParser::Pos & pos, Expected & expected, ASTGrantQuery::Kind kind, std::shared_ptr & to_roles) { return IParserBase::wrapParseImpl(pos, [&] { @@ -230,10 +232,12 @@ namespace } ASTPtr ast; - if (!ParserExtendedRoleSet{}.enableAllKeyword(kind == Kind::REVOKE).parse(pos, ast, expected)) + ParserRolesOrUsersSet roles_p; + roles_p.allowRoleNames().allowUserNames().allowCurrentUser().allowAll(kind == Kind::REVOKE); + if (!roles_p.parse(pos, ast, expected)) return false; - to_roles = typeid_cast>(ast); + to_roles = typeid_cast>(ast); return true; }); } @@ -282,14 +286,14 @@ bool ParserGrantQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } AccessRightsElements elements; - std::shared_ptr roles; + std::shared_ptr roles; if (!parseAccessRightsElements(pos, expected, elements) && !parseRoles(pos, expected, attach, roles)) return false; if (cluster.empty()) parseOnCluster(pos, expected, cluster); - std::shared_ptr to_roles; + std::shared_ptr to_roles; if (!parseToRoles(pos, expected, kind, to_roles)) return false; diff --git a/src/Parsers/ParserGrantQuery.h b/src/Parsers/ParserGrantQuery.h index ac5bfbb7538..b14f175c12b 100644 --- a/src/Parsers/ParserGrantQuery.h +++ b/src/Parsers/ParserGrantQuery.h @@ -12,7 +12,7 @@ namespace DB class ParserGrantQuery : public IParserBase { public: - ParserGrantQuery & enableAttachMode(bool enable) { attach_mode = enable; return *this; } + ParserGrantQuery & useAttachMode(bool attach_mode_ = true) { attach_mode = attach_mode_; return *this; } protected: const char * getName() const override { return "GRANT or REVOKE query"; } diff --git a/src/Parsers/ParserExtendedRoleSet.cpp b/src/Parsers/ParserRolesOrUsersSet.cpp similarity index 81% rename from src/Parsers/ParserExtendedRoleSet.cpp rename to src/Parsers/ParserRolesOrUsersSet.cpp index 80f05c45f5b..1ba2c05f671 100644 --- a/src/Parsers/ParserExtendedRoleSet.cpp +++ b/src/Parsers/ParserRolesOrUsersSet.cpp @@ -1,8 +1,8 @@ -#include +#include #include #include #include -#include +#include #include #include @@ -39,8 +39,8 @@ namespace IParserBase::Pos & pos, Expected & expected, bool id_mode, - bool all_keyword_enabled, - bool current_user_keyword_enabled, + bool allow_all, + bool allow_current_user_tag, Strings & names, bool & all, bool & current_user) @@ -56,7 +56,7 @@ namespace { } else if ( - current_user_keyword_enabled + allow_current_user_tag && (ParserKeyword{"CURRENT_USER"}.ignore(pos, expected) || ParserKeyword{"currentUser"}.ignore(pos, expected))) { if (ParserToken{TokenType::OpeningRoundBracket}.ignore(pos, expected)) @@ -66,7 +66,7 @@ namespace } res_current_user = true; } - else if (all_keyword_enabled && ParserKeyword{"ALL"}.ignore(pos, expected)) + else if (allow_all && ParserKeyword{"ALL"}.ignore(pos, expected)) { res_all = true; } @@ -93,7 +93,7 @@ namespace IParserBase::Pos & pos, Expected & expected, bool id_mode, - bool current_user_keyword_enabled, + bool allow_current_user_tag, Strings & except_names, bool & except_current_user) { @@ -103,13 +103,13 @@ namespace return false; bool dummy; - return parseBeforeExcept(pos, expected, id_mode, false, current_user_keyword_enabled, except_names, dummy, except_current_user); + return parseBeforeExcept(pos, expected, id_mode, false, allow_current_user_tag, except_names, dummy, except_current_user); }); } } -bool ParserExtendedRoleSet::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +bool ParserRolesOrUsersSet::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { Strings names; bool current_user = false; @@ -117,21 +117,23 @@ bool ParserExtendedRoleSet::parseImpl(Pos & pos, ASTPtr & node, Expected & expec Strings except_names; bool except_current_user = false; - if (!parseBeforeExcept(pos, expected, id_mode, all_keyword, current_user_keyword, names, all, current_user)) + if (!parseBeforeExcept(pos, expected, id_mode, allow_all, allow_current_user, names, all, current_user)) return false; - parseExceptAndAfterExcept(pos, expected, id_mode, current_user_keyword, except_names, except_current_user); + parseExceptAndAfterExcept(pos, expected, id_mode, allow_current_user, except_names, except_current_user); if (all) names.clear(); - auto result = std::make_shared(); + auto result = std::make_shared(); result->names = std::move(names); result->current_user = current_user; result->all = all; result->except_names = std::move(except_names); result->except_current_user = except_current_user; result->id_mode = id_mode; + result->allow_user_names = allow_user_names; + result->allow_role_names = allow_role_names; node = result; return true; } diff --git a/src/Parsers/ParserRolesOrUsersSet.h b/src/Parsers/ParserRolesOrUsersSet.h new file mode 100644 index 00000000000..c71012e874c --- /dev/null +++ b/src/Parsers/ParserRolesOrUsersSet.h @@ -0,0 +1,32 @@ +#pragma once + +#include + + +namespace DB +{ +/** Parses a string like this: + * {role|CURRENT_USER} [,...] | NONE | ALL | ALL EXCEPT {role|CURRENT_USER} [,...] + */ +class ParserRolesOrUsersSet : public IParserBase +{ +public: + ParserRolesOrUsersSet & allowAll(bool allow_all_ = true) { allow_all = allow_all_; return *this; } + ParserRolesOrUsersSet & allowUserNames(bool allow_user_names_ = true) { allow_user_names = allow_user_names_; return *this; } + ParserRolesOrUsersSet & allowRoleNames(bool allow_role_names_ = true) { allow_role_names = allow_role_names_; return *this; } + ParserRolesOrUsersSet & allowCurrentUser(bool allow_current_user_ = true) { allow_current_user = allow_current_user_; return *this; } + ParserRolesOrUsersSet & useIDMode(bool id_mode_ = true) { id_mode = id_mode_; return *this; } + +protected: + const char * getName() const override { return "RolesOrUsersSet"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +private: + bool allow_all = false; + bool allow_user_names = false; + bool allow_role_names = false; + bool allow_current_user = false; + bool id_mode = false; +}; + +} diff --git a/src/Parsers/ParserSetRoleQuery.cpp b/src/Parsers/ParserSetRoleQuery.cpp index a69480f89eb..e8734f8dfc1 100644 --- a/src/Parsers/ParserSetRoleQuery.cpp +++ b/src/Parsers/ParserSetRoleQuery.cpp @@ -1,29 +1,31 @@ #include #include #include -#include -#include +#include +#include namespace DB { namespace { - bool parseRoles(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & roles) + bool parseRoles(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & roles) { return IParserBase::wrapParseImpl(pos, [&] { ASTPtr ast; - if (!ParserExtendedRoleSet{}.enableCurrentUserKeyword(false).parse(pos, ast, expected)) + ParserRolesOrUsersSet roles_p; + roles_p.allowRoleNames().allowAll(); + if (!roles_p.parse(pos, ast, expected)) return false; - roles = typeid_cast>(ast); - roles->can_contain_users = false; + roles = typeid_cast>(ast); + roles->allow_user_names = false; return true; }); } - bool parseToUsers(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & to_users) + bool parseToUsers(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & to_users) { return IParserBase::wrapParseImpl(pos, [&] { @@ -31,11 +33,13 @@ namespace return false; ASTPtr ast; - if (!ParserExtendedRoleSet{}.enableAllKeyword(false).parse(pos, ast, expected)) + ParserRolesOrUsersSet users_p; + users_p.allowUserNames().allowCurrentUser(); + if (!users_p.parse(pos, ast, expected)) return false; - to_users = typeid_cast>(ast); - to_users->can_contain_roles = false; + to_users = typeid_cast>(ast); + to_users->allow_role_names = false; return true; }); } @@ -55,8 +59,8 @@ bool ParserSetRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected else return false; - std::shared_ptr roles; - std::shared_ptr to_users; + std::shared_ptr roles; + std::shared_ptr to_users; if ((kind == Kind::SET_ROLE) || (kind == Kind::SET_DEFAULT_ROLE)) { diff --git a/src/Parsers/ParserSettingsProfileElement.cpp b/src/Parsers/ParserSettingsProfileElement.cpp index 1dccae50cf5..2dd65e6ae7b 100644 --- a/src/Parsers/ParserSettingsProfileElement.cpp +++ b/src/Parsers/ParserSettingsProfileElement.cpp @@ -109,7 +109,7 @@ bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected std::optional readonly; if (ParserKeyword{"PROFILE"}.ignore(pos, expected) || - (enable_inherit_keyword && ParserKeyword{"INHERIT"}.ignore(pos, expected))) + (use_inherit_keyword && ParserKeyword{"INHERIT"}.ignore(pos, expected))) { if (!parseProfileNameOrID(pos, expected, id_mode, parent_profile)) return false; @@ -140,7 +140,7 @@ bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected result->max_value = std::move(max_value); result->readonly = readonly; result->id_mode = id_mode; - result->use_inherit_keyword = enable_inherit_keyword; + result->use_inherit_keyword = use_inherit_keyword; node = result; return true; } @@ -155,10 +155,12 @@ bool ParserSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Expected } else { + ParserSettingsProfileElement element_p; + element_p.useIDMode(id_mode).useInheritKeyword(use_inherit_keyword); do { ASTPtr ast; - if (!ParserSettingsProfileElement{}.useIDMode(id_mode).enableInheritKeyword(enable_inherit_keyword).parse(pos, ast, expected)) + if (!element_p.parse(pos, ast, expected)) return false; auto element = typeid_cast>(ast); elements.push_back(std::move(element)); diff --git a/src/Parsers/ParserSettingsProfileElement.h b/src/Parsers/ParserSettingsProfileElement.h index 309c797e645..8843591a56c 100644 --- a/src/Parsers/ParserSettingsProfileElement.h +++ b/src/Parsers/ParserSettingsProfileElement.h @@ -11,8 +11,8 @@ namespace DB class ParserSettingsProfileElement : public IParserBase { public: - ParserSettingsProfileElement & useIDMode(bool enable_) { id_mode = enable_; return *this; } - ParserSettingsProfileElement & enableInheritKeyword(bool enable_) { enable_inherit_keyword = enable_; return *this; } + ParserSettingsProfileElement & useIDMode(bool id_mode_ = true) { id_mode = id_mode_; return *this; } + ParserSettingsProfileElement & useInheritKeyword(bool use_inherit_keyword_ = true) { use_inherit_keyword = use_inherit_keyword_; return *this; } protected: const char * getName() const override { return "SettingsProfileElement"; } @@ -20,15 +20,15 @@ protected: private: bool id_mode = false; - bool enable_inherit_keyword = false; + bool use_inherit_keyword = false; }; class ParserSettingsProfileElements : public IParserBase { public: - ParserSettingsProfileElements & useIDMode(bool enable_) { id_mode = enable_; return *this; } - ParserSettingsProfileElements & enableInheritKeyword(bool enable_) { enable_inherit_keyword = enable_; return *this; } + ParserSettingsProfileElements & useIDMode(bool id_mode_ = true) { id_mode = id_mode_; return *this; } + ParserSettingsProfileElements & useInheritKeyword(bool use_inherit_keyword_ = true) { use_inherit_keyword = use_inherit_keyword_; return *this; } protected: const char * getName() const override { return "SettingsProfileElements"; } @@ -36,7 +36,7 @@ protected: private: bool id_mode = false; - bool enable_inherit_keyword = false; + bool use_inherit_keyword = false; }; } diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index 60672c0c116..fd60c2f3551 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -21,7 +21,6 @@ SRCS( ASTDropAccessEntityQuery.cpp ASTDropQuery.cpp ASTExpressionList.cpp - ASTExtendedRoleSet.cpp ASTFunction.cpp ASTFunctionWithKeyValueArguments.cpp ASTGrantQuery.cpp @@ -37,6 +36,7 @@ SRCS( ASTQueryWithOnCluster.cpp ASTQueryWithOutput.cpp ASTQueryWithTableAndOutput.cpp + ASTRolesOrUsersSet.cpp ASTRowPolicyName.cpp ASTSampleRatio.cpp ASTSelectQuery.cpp @@ -81,7 +81,6 @@ SRCS( ParserDictionaryAttributeDeclaration.cpp ParserDropAccessEntityQuery.cpp ParserDropQuery.cpp - ParserExtendedRoleSet.cpp ParserGrantQuery.cpp ParserInsertQuery.cpp ParserKillQueryQuery.cpp @@ -90,6 +89,7 @@ SRCS( ParserQuery.cpp ParserQueryWithOutput.cpp ParserRenameQuery.cpp + ParserRolesOrUsersSet.cpp ParserRowPolicyName.cpp ParserSampleRatio.cpp ParserSelectQuery.cpp diff --git a/src/Storages/System/StorageSystemPrivileges.cpp b/src/Storages/System/StorageSystemPrivileges.cpp index f0a1b21368e..5dda0caf201 100644 --- a/src/Storages/System/StorageSystemPrivileges.cpp +++ b/src/Storages/System/StorageSystemPrivileges.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemQuotas.cpp b/src/Storages/System/StorageSystemQuotas.cpp index a3b687dc011..5d8c0be5861 100644 --- a/src/Storages/System/StorageSystemQuotas.cpp +++ b/src/Storages/System/StorageSystemQuotas.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include @@ -72,7 +72,7 @@ void StorageSystemQuotas::fillData(MutableColumns & res_columns, const Context & const String & storage_name, const std::vector & all_limits, KeyType key_type, - const ExtendedRoleSet & apply_to) + const RolesOrUsersSet & apply_to) { column_name.insertData(name.data(), name.length()); column_id.push_back(id); diff --git a/src/Storages/System/StorageSystemRoleGrants.cpp b/src/Storages/System/StorageSystemRoleGrants.cpp index 00147a0dae6..0f0fcd831d9 100644 --- a/src/Storages/System/StorageSystemRoleGrants.cpp +++ b/src/Storages/System/StorageSystemRoleGrants.cpp @@ -78,7 +78,7 @@ void StorageSystemRoleGrants::fillData(MutableColumns & res_columns, const Conte auto add_rows = [&](const String & grantee_name, IAccessEntity::Type grantee_type, const GrantedRoles & granted_roles, - const ExtendedRoleSet * default_roles) + const RolesOrUsersSet * default_roles) { for (const auto & role_id : granted_roles.roles) { @@ -99,7 +99,7 @@ void StorageSystemRoleGrants::fillData(MutableColumns & res_columns, const Conte continue; const GrantedRoles * granted_roles = nullptr; - const ExtendedRoleSet * default_roles = nullptr; + const RolesOrUsersSet * default_roles = nullptr; if (auto role = typeid_cast(entity)) granted_roles = &role->granted_roles; else if (auto user = typeid_cast(entity)) diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index ca77a5182a5..9f5267b3a9b 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include @@ -87,7 +87,7 @@ void StorageSystemRowPolicies::fillData(MutableColumns & res_columns, const Cont const String & storage_name, const std::array & conditions, bool is_restrictive, - const ExtendedRoleSet & apply_to) + const RolesOrUsersSet & apply_to) { column_name.insertData(name.data(), name.length()); column_short_name.insertData(name_parts.short_name.data(), name_parts.short_name.length()); diff --git a/src/Storages/System/StorageSystemSettingsProfiles.cpp b/src/Storages/System/StorageSystemSettingsProfiles.cpp index d02c5910608..610e5bb68f5 100644 --- a/src/Storages/System/StorageSystemSettingsProfiles.cpp +++ b/src/Storages/System/StorageSystemSettingsProfiles.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include @@ -51,7 +51,7 @@ void StorageSystemSettingsProfiles::fillData(MutableColumns & res_columns, const const UUID & id, const String & storage_name, const SettingsProfileElements & elements, - const ExtendedRoleSet & apply_to) + const RolesOrUsersSet & apply_to) { column_name.insertData(name.data(), name.length()); column_id.push_back(id); diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index e0755fe59ab..7f3fe058d9e 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include @@ -80,7 +80,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & const String & storage_name, const Authentication & authentication, const AllowedClientHosts & allowed_hosts, - const ExtendedRoleSet & default_roles) + const RolesOrUsersSet & default_roles) { column_name.insertData(name.data(), name.length()); column_id.push_back(id); From 7d1951a79b6d5b6826cb0da7e4fb35ee18e8dc09 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 1 Jun 2020 13:45:07 +0300 Subject: [PATCH 226/329] Improve messages for errors in access storages. --- src/Access/IAccessStorage.cpp | 328 ++++++++++++++++++---------------- 1 file changed, 169 insertions(+), 159 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index a7af61c7712..6813b5eb558 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -24,16 +24,141 @@ namespace using EntityType = IAccessStorage::EntityType; using EntityTypeInfo = IAccessStorage::EntityTypeInfo; - bool isNotFoundErrorCode(int error_code) + + String outputID(const UUID & id) { - if (error_code == ErrorCodes::ACCESS_ENTITY_NOT_FOUND) - return true; + return "ID(" + toString(id) + ")"; + } - for (auto type : ext::range(EntityType::MAX)) - if (error_code == EntityTypeInfo::get(type).not_found_error_code) - return true; + String outputTypeAndNameOrID(const IAccessStorage & storage, const UUID & id) + { + auto entity = storage.tryRead(id); + if (entity) + return entity->outputTypeAndName(); + return outputID(id); + } - return false; + + template > + ResultType doTry(const Func & func) + { + try + { + return func(); + } + catch (Exception &) + { + return {}; + } + } + + + template , + typename ResultType = std::conditional_t, void, std::vector>> + ResultType applyToMultipleEntities( + const std::vector & multiple_entities, + const ApplyFunc & apply_function, + const char * error_message_format [[maybe_unused]] = nullptr, + const GetNameFunc & get_name_function [[maybe_unused]] = nullptr) + { + std::optional exception; + std::vector success; + + auto helper = [&](const auto & apply_and_store_result_function) + { + for (size_t i = 0; i != multiple_entities.size(); ++i) + { + try + { + apply_and_store_result_function(multiple_entities[i]); + if constexpr (!ignore_errors) + success[i] = true; + } + catch (Exception & e) + { + if (!ignore_errors && !exception) + exception.emplace(e); + } + catch (Poco::Exception & e) + { + if (!ignore_errors && !exception) + exception.emplace(Exception::CreateFromPocoTag{}, e); + } + catch (std::exception & e) + { + if (!ignore_errors && !exception) + exception.emplace(Exception::CreateFromSTDTag{}, e); + } + } + }; + + if constexpr (std::is_same_v) + { + if (multiple_entities.empty()) + return; + + if (multiple_entities.size() == 1) + { + apply_function(multiple_entities.front()); + return; + } + + if constexpr (!ignore_errors) + success.resize(multiple_entities.size(), false); + + helper(apply_function); + + if (ignore_errors || !exception) + return; + } + else + { + ResultType result; + if (multiple_entities.empty()) + return result; + + if (multiple_entities.size() == 1) + { + result.emplace_back(apply_function(multiple_entities.front())); + return result; + } + + result.reserve(multiple_entities.size()); + if constexpr (!ignore_errors) + success.resize(multiple_entities.size(), false); + + helper([&](const T & entity) { result.emplace_back(apply_function(entity)); }); + + if (ignore_errors || !exception) + return result; + } + + if constexpr (!ignore_errors) + { + Strings succeeded_names_list; + Strings failed_names_list; + for (size_t i = 0; i != multiple_entities.size(); ++i) + { + const auto & entity = multiple_entities[i]; + String name = get_name_function(entity); + if (success[i]) + succeeded_names_list.emplace_back(name); + else + failed_names_list.emplace_back(name); + } + String succeeded_names = boost::algorithm::join(succeeded_names_list, ", "); + String failed_names = boost::algorithm::join(failed_names_list, ", "); + if (succeeded_names.empty()) + succeeded_names = "none"; + + String error_message = error_message_format; + boost::replace_all(error_message, "{succeeded_names}", succeeded_names); + boost::replace_all(error_message, "{failed_names}", failed_names); + exception->addMessage(error_message); + exception->rethrow(); + } + __builtin_unreachable(); } } @@ -91,14 +216,7 @@ bool IAccessStorage::exists(const UUID & id) const AccessEntityPtr IAccessStorage::tryReadBase(const UUID & id) const { - try - { - return readImpl(id); - } - catch (Exception &) - { - return nullptr; - } + return doTry([&] { return readImpl(id); }); } @@ -110,14 +228,7 @@ String IAccessStorage::readName(const UUID & id) const std::optional IAccessStorage::tryReadName(const UUID & id) const { - try - { - return readNameImpl(id); - } - catch (Exception &) - { - return {}; - } + return doTry([&] { return std::optional{readNameImpl(id)}; }); } @@ -129,56 +240,25 @@ UUID IAccessStorage::insert(const AccessEntityPtr & entity) std::vector IAccessStorage::insert(const std::vector & multiple_entities) { - std::vector ids; - ids.reserve(multiple_entities.size()); - String error_message; - for (const auto & entity : multiple_entities) - { - try - { - ids.push_back(insertImpl(entity, false)); - } - catch (Exception & e) - { - if (e.code() != ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS) - throw; - error_message += (error_message.empty() ? "" : ". ") + e.message(); - } - } - if (!error_message.empty()) - throw Exception(error_message, ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); - return ids; + return applyToMultipleEntities( + multiple_entities, + [this](const AccessEntityPtr & entity) { return insertImpl(entity, /* replace_if_exists = */ false); }, + "Couldn't insert {failed_names}. Successfully inserted: {succeeded_names}", + [](const AccessEntityPtr & entity) { return entity->outputTypeAndName(); }); } std::optional IAccessStorage::tryInsert(const AccessEntityPtr & entity) { - try - { - return insertImpl(entity, false); - } - catch (Exception &) - { - return {}; - } + return doTry([&] { return std::optional{insertImpl(entity, false)}; }); } std::vector IAccessStorage::tryInsert(const std::vector & multiple_entities) { - std::vector ids; - ids.reserve(multiple_entities.size()); - for (const auto & entity : multiple_entities) - { - try - { - ids.push_back(insertImpl(entity, false)); - } - catch (Exception &) - { - } - } - return ids; + return applyToMultipleEntities( + multiple_entities, + [this](const AccessEntityPtr & entity) { return insertImpl(entity, /* replace_if_exists = */ false); }); } @@ -190,11 +270,11 @@ UUID IAccessStorage::insertOrReplace(const AccessEntityPtr & entity) std::vector IAccessStorage::insertOrReplace(const std::vector & multiple_entities) { - std::vector ids; - ids.reserve(multiple_entities.size()); - for (const auto & entity : multiple_entities) - ids.push_back(insertImpl(entity, true)); - return ids; + return applyToMultipleEntities( + multiple_entities, + [this](const AccessEntityPtr & entity) { return insertImpl(entity, /* replace_if_exists = */ true); }, + "Couldn't insert {failed_names}. Successfully inserted: {succeeded_names}", + [](const AccessEntityPtr & entity) -> String { return entity->outputTypeAndName(); }); } @@ -206,60 +286,25 @@ void IAccessStorage::remove(const UUID & id) void IAccessStorage::remove(const std::vector & ids) { - String error_message; - std::optional error_code; - for (const auto & id : ids) - { - try - { - removeImpl(id); - } - catch (Exception & e) - { - if (!isNotFoundErrorCode(e.code())) - throw; - error_message += (error_message.empty() ? "" : ". ") + e.message(); - if (error_code && (*error_code != e.code())) - error_code = ErrorCodes::ACCESS_ENTITY_NOT_FOUND; - else - error_code = e.code(); - } - } - if (!error_message.empty()) - throw Exception(error_message, *error_code); + applyToMultipleEntities( + ids, + [this](const UUID & id) { removeImpl(id); }, + "Couldn't remove {failed_names}. Successfully removed: {succeeded_names}", + [this](const UUID & id) { return outputTypeAndNameOrID(*this, id); }); } bool IAccessStorage::tryRemove(const UUID & id) { - try - { - removeImpl(id); - return true; - } - catch (Exception &) - { - return false; - } + return doTry([&] { removeImpl(id); return true; }); } std::vector IAccessStorage::tryRemove(const std::vector & ids) { - std::vector removed; - removed.reserve(ids.size()); - for (const auto & id : ids) - { - try - { - removeImpl(id); - removed.push_back(id); - } - catch (Exception &) - { - } - } - return removed; + return applyToMultipleEntities( + ids, + [this](const UUID & id) { removeImpl(id); return id; }); } @@ -271,60 +316,25 @@ void IAccessStorage::update(const UUID & id, const UpdateFunc & update_func) void IAccessStorage::update(const std::vector & ids, const UpdateFunc & update_func) { - String error_message; - std::optional error_code; - for (const auto & id : ids) - { - try - { - updateImpl(id, update_func); - } - catch (Exception & e) - { - if (!isNotFoundErrorCode(e.code())) - throw; - error_message += (error_message.empty() ? "" : ". ") + e.message(); - if (error_code && (*error_code != e.code())) - error_code = ErrorCodes::ACCESS_ENTITY_NOT_FOUND; - else - error_code = e.code(); - } - } - if (!error_message.empty()) - throw Exception(error_message, *error_code); + applyToMultipleEntities( + ids, + [this, &update_func](const UUID & id) { updateImpl(id, update_func); }, + "Couldn't update {failed_names}. Successfully updated: {succeeded_names}", + [this](const UUID & id) { return outputTypeAndNameOrID(*this, id); }); } bool IAccessStorage::tryUpdate(const UUID & id, const UpdateFunc & update_func) { - try - { - updateImpl(id, update_func); - return true; - } - catch (Exception &) - { - return false; - } + return doTry([&] { updateImpl(id, update_func); return true; }); } std::vector IAccessStorage::tryUpdate(const std::vector & ids, const UpdateFunc & update_func) { - std::vector updated; - updated.reserve(ids.size()); - for (const auto & id : ids) - { - try - { - updateImpl(id, update_func); - updated.push_back(id); - } - catch (Exception &) - { - } - } - return updated; + return applyToMultipleEntities( + ids, + [this, &update_func](const UUID & id) { updateImpl(id, update_func); return id; }); } @@ -388,7 +398,7 @@ Poco::Logger * IAccessStorage::getLogger() const void IAccessStorage::throwNotFound(const UUID & id) const { - throw Exception("ID {" + toString(id) + "} not found in [" + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_NOT_FOUND); + throw Exception(outputID(id) + " not found in [" + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_NOT_FOUND); } @@ -402,7 +412,7 @@ void IAccessStorage::throwNotFound(EntityType type, const String & name) const void IAccessStorage::throwBadCast(const UUID & id, EntityType type, const String & name, EntityType required_type) { throw Exception( - "ID {" + toString(id) + "}: " + outputEntityTypeAndName(type, name) + " expected to be of type " + toString(required_type), + outputID(id) + ": " + outputEntityTypeAndName(type, name) + " expected to be of type " + toString(required_type), ErrorCodes::LOGICAL_ERROR); } @@ -410,7 +420,7 @@ void IAccessStorage::throwBadCast(const UUID & id, EntityType type, const String void IAccessStorage::throwIDCollisionCannotInsert(const UUID & id, EntityType type, const String & name, EntityType existing_type, const String & existing_name) const { throw Exception( - outputEntityTypeAndName(type, name) + ": cannot insert because the ID {" + toString(id) + "} is already used by " + outputEntityTypeAndName(type, name) + ": cannot insert because the " + outputID(id) + " is already used by " + outputEntityTypeAndName(existing_type, existing_name) + " in [" + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); } From 4bd00b02e217be02ed128ce78bc13b12db62dc69 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 5 Jun 2020 20:57:33 +0300 Subject: [PATCH 227/329] Improve syntax of CREATE QUOTA. Now resource types and key types could be written with underscores. Also rename columns key_type=>keys and source=>storage in table system.quotas. --- src/Access/Quota.h | 22 ++- src/Common/IntervalKind.cpp | 17 ++ src/Common/IntervalKind.h | 2 + ...InterpreterShowCreateAccessEntityQuery.cpp | 4 +- src/Parsers/ASTCreateQuotaQuery.cpp | 65 +++++--- src/Parsers/ASTCreateQuotaQuery.h | 12 +- src/Parsers/ParserCreateQuotaQuery.cpp | 149 ++++++++++++------ src/Parsers/ParserCreateQuotaQuery.h | 12 +- src/Storages/System/StorageSystemQuotas.cpp | 24 ++- .../test_disk_access_storage/test.py | 2 +- tests/integration/test_quota/test.py | 49 +++--- .../0_stateless/01033_quota_dcl.reference | 2 +- 12 files changed, 244 insertions(+), 116 deletions(-) diff --git a/src/Access/Quota.h b/src/Access/Quota.h index 101263e76a5..5bbea36cfda 100644 --- a/src/Access/Quota.h +++ b/src/Access/Quota.h @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include @@ -84,7 +86,8 @@ struct Quota : public IAccessEntity struct KeyTypeInfo { const char * const raw_name; - const String name; /// Lowercased with spaces, e.g. "client key". + const String name; /// Lowercased with underscores, e.g. "client_key". + const std::vector base_types; /// For combined types keeps base types, e.g. for CLIENT_KEY_OR_USER_NAME it keeps [KeyType::CLIENT_KEY, KeyType::USER_NAME]. static const KeyTypeInfo & get(KeyType type); }; @@ -195,8 +198,21 @@ inline const Quota::KeyTypeInfo & Quota::KeyTypeInfo::get(KeyType type) { String init_name = raw_name_; boost::to_lower(init_name); - boost::replace_all(init_name, "_", " "); - return KeyTypeInfo{raw_name_, std::move(init_name)}; + std::vector init_base_types; + String replaced = boost::algorithm::replace_all_copy(init_name, "_or_", "|"); + Strings tokens; + boost::algorithm::split(tokens, replaced, boost::is_any_of("|")); + if (tokens.size() > 1) + { + for (const auto & token : tokens) + for (auto kt : ext::range(KeyType::MAX)) + if (KeyTypeInfo::get(kt).name == token) + { + init_base_types.push_back(kt); + break; + } + } + return KeyTypeInfo{raw_name_, std::move(init_name), std::move(init_base_types)}; }; switch (type) diff --git a/src/Common/IntervalKind.cpp b/src/Common/IntervalKind.cpp index 9443844a54b..1582889eff2 100644 --- a/src/Common/IntervalKind.cpp +++ b/src/Common/IntervalKind.cpp @@ -83,6 +83,23 @@ const char * IntervalKind::toKeyword() const } +const char * IntervalKind::toLowercasedKeyword() const +{ + switch (kind) + { + case IntervalKind::Second: return "second"; + case IntervalKind::Minute: return "minute"; + case IntervalKind::Hour: return "hour"; + case IntervalKind::Day: return "day"; + case IntervalKind::Week: return "week"; + case IntervalKind::Month: return "month"; + case IntervalKind::Quarter: return "quarter"; + case IntervalKind::Year: return "year"; + } + __builtin_unreachable(); +} + + const char * IntervalKind::toDateDiffUnit() const { switch (kind) diff --git a/src/Common/IntervalKind.h b/src/Common/IntervalKind.h index 9b7c4bd504e..d8a569b8de4 100644 --- a/src/Common/IntervalKind.h +++ b/src/Common/IntervalKind.h @@ -37,6 +37,8 @@ struct IntervalKind /// Returns an uppercased version of what `toString()` returns. const char * toKeyword() const; + const char * toLowercasedKeyword() const; + /// Returns the string which can be passed to the `unit` parameter of the dateDiff() function. /// For example, `IntervalKind{IntervalKind::Day}.getDateDiffParameter()` returns "day". const char * toDateDiffUnit() const; diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 9c28c3d0bd2..18a08d21e93 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -132,7 +132,9 @@ namespace query->names.emplace_back(quota.getName()); query->attach = attach_mode; - query->key_type = quota.key_type; + if (quota.key_type != Quota::KeyType::NONE) + query->key_type = quota.key_type; + query->all_limits.reserve(quota.all_limits.size()); for (const auto & limits : quota.all_limits) diff --git a/src/Parsers/ASTCreateQuotaQuery.cpp b/src/Parsers/ASTCreateQuotaQuery.cpp index fc4e2edb9e7..88516fb6eac 100644 --- a/src/Parsers/ASTCreateQuotaQuery.cpp +++ b/src/Parsers/ASTCreateQuotaQuery.cpp @@ -18,8 +18,28 @@ namespace void formatKeyType(const KeyType & key_type, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " KEYED BY " << (settings.hilite ? IAST::hilite_none : "") << "'" - << KeyTypeInfo::get(key_type).name << "'"; + const auto & type_info = KeyTypeInfo::get(key_type); + if (key_type == KeyType::NONE) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT KEYED" << (settings.hilite ? IAST::hilite_none : ""); + return; + } + + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " KEYED BY " << (settings.hilite ? IAST::hilite_none : ""); + + if (!type_info.base_types.empty()) + { + bool need_comma = false; + for (const auto & base_type : type_info.base_types) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << KeyTypeInfo::get(base_type).name; + } + return; + } + + settings.ostr << type_info.name; } @@ -43,20 +63,14 @@ namespace } - void formatLimit(ResourceType resource_type, ResourceAmount max, bool first, const IAST::FormatSettings & settings) + void formatLimit(ResourceType resource_type, ResourceAmount max, const IAST::FormatSettings & settings) { - if (first) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MAX" << (settings.hilite ? IAST::hilite_none : ""); - else - settings.ostr << ","; - const auto & type_info = ResourceTypeInfo::get(resource_type); - settings.ostr << " " << (settings.hilite ? IAST::hilite_keyword : "") << type_info.keyword - << (settings.hilite ? IAST::hilite_none : "") << " " << type_info.amountToString(max); + settings.ostr << " " << type_info.name << " = " << type_info.amountToString(max); } - void formatLimits(const ASTCreateQuotaQuery::Limits & limits, const IAST::FormatSettings & settings) + void formatIntervalWithLimits(const ASTCreateQuotaQuery::Limits & limits, const IAST::FormatSettings & settings) { auto interval_kind = IntervalKind::fromAvgSeconds(limits.duration.count()); Int64 num_intervals = limits.duration.count() / interval_kind.toAvgSeconds(); @@ -64,11 +78,11 @@ namespace settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " FOR" << (limits.randomize_interval ? " RANDOMIZED" : "") - << " INTERVAL " + << " INTERVAL" << (settings.hilite ? IAST::hilite_none : "") - << num_intervals << " " + << " " << num_intervals << " " << (settings.hilite ? IAST::hilite_keyword : "") - << interval_kind.toKeyword() + << interval_kind.toLowercasedKeyword() << (settings.hilite ? IAST::hilite_none : ""); if (limits.drop) @@ -81,17 +95,28 @@ namespace for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) { if (limits.max[resource_type]) - { - formatLimit(resource_type, *limits.max[resource_type], !limit_found, settings); limit_found = true; + } + if (limit_found) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " MAX" << (settings.hilite ? IAST::hilite_none : ""); + bool need_comma = false; + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + { + if (limits.max[resource_type]) + { + if (std::exchange(need_comma, true)) + settings.ostr << ","; + formatLimit(resource_type, *limits.max[resource_type], settings); + } } } - if (!limit_found) + else settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TRACKING ONLY" << (settings.hilite ? IAST::hilite_none : ""); } } - void formatAllLimits(const std::vector & all_limits, const IAST::FormatSettings & settings) + void formatIntervalsWithLimits(const std::vector & all_limits, const IAST::FormatSettings & settings) { bool need_comma = false; for (const auto & limits : all_limits) @@ -100,7 +125,7 @@ namespace settings.ostr << ","; need_comma = true; - formatLimits(limits, settings); + formatIntervalWithLimits(limits, settings); } } @@ -152,7 +177,7 @@ void ASTCreateQuotaQuery::formatImpl(const FormatSettings & settings, FormatStat if (key_type) formatKeyType(*key_type, settings); - formatAllLimits(all_limits, settings); + formatIntervalsWithLimits(all_limits, settings); if (roles && (!roles->empty() || alter)) formatToRoles(*roles, settings); diff --git a/src/Parsers/ASTCreateQuotaQuery.h b/src/Parsers/ASTCreateQuotaQuery.h index 002c374322f..1671ae1d00f 100644 --- a/src/Parsers/ASTCreateQuotaQuery.h +++ b/src/Parsers/ASTCreateQuotaQuery.h @@ -11,17 +11,17 @@ class ASTRolesOrUsersSet; /** CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name - * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] - * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} - * {MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number} [,...] | + * [KEYED BY {none | user_name | ip_address | client_key | client_key, user_name | client_key, ip_address} | NOT KEYED] + * [FOR [RANDOMIZED] INTERVAL number {second | minute | hour | day} + * {MAX {{queries | errors | result_rows | result_bytes | read_rows | read_bytes | execution_time} = number} [,...] | * NO LIMITS | TRACKING ONLY} [,...]] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] * * ALTER QUOTA [IF EXISTS] name * [RENAME TO new_name] - * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] - * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} - * {MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number} [,...] | + * [KEYED BY {none | user_name | ip_address | client_key | client_key, user_name | client_key, ip_address} | NOT KEYED] + * [FOR [RANDOMIZED] INTERVAL number {second | minute | hour | day} + * {MAX {{queries | errors | result_rows | result_bytes | read_rows | read_bytes | execution_time} = number} [,...] | * NO LIMITS | TRACKING ONLY} [,...]] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] */ diff --git a/src/Parsers/ParserCreateQuotaQuery.cpp b/src/Parsers/ParserCreateQuotaQuery.cpp index e953f698de0..f83bac975b0 100644 --- a/src/Parsers/ParserCreateQuotaQuery.cpp +++ b/src/Parsers/ParserCreateQuotaQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -39,84 +40,126 @@ namespace }); } - bool parseKeyType(IParserBase::Pos & pos, Expected & expected, std::optional & key_type) + bool parseKeyType(IParserBase::Pos & pos, Expected & expected, KeyType & key_type) { return IParserBase::wrapParseImpl(pos, [&] { - if (!ParserKeyword{"KEYED BY"}.ignore(pos, expected)) + if (ParserKeyword{"NOT KEYED"}.ignore(pos, expected)) + { + key_type = KeyType::NONE; + return true; + } + + if (!ParserKeyword{"KEY BY"}.ignore(pos, expected) && !ParserKeyword{"KEYED BY"}.ignore(pos, expected)) return false; - ASTPtr key_type_ast; - if (!ParserStringLiteral().parse(pos, key_type_ast, expected)) + Strings names; + if (!parseIdentifiersOrStringLiterals(pos, expected, names)) return false; - const String & key_type_str = key_type_ast->as().value.safeGet(); + String name = boost::algorithm::join(names, "_or_"); + boost::to_lower(name); + boost::replace_all(name, " ", "_"); + for (auto kt : ext::range(Quota::KeyType::MAX)) - if (boost::iequals(KeyTypeInfo::get(kt).name, key_type_str)) + if (KeyTypeInfo::get(kt).name == name) { key_type = kt; return true; } - String all_key_types_str; + String all_types_str; for (auto kt : ext::range(Quota::KeyType::MAX)) - all_key_types_str += String(all_key_types_str.empty() ? "" : ", ") + "'" + KeyTypeInfo::get(kt).name + "'"; - String msg = "Quota cannot be keyed by '" + key_type_str + "'. Expected one of these literals: " + all_key_types_str; + all_types_str += String(all_types_str.empty() ? "" : ", ") + "'" + KeyTypeInfo::get(kt).name + "'"; + String msg = "Quota cannot be keyed by '" + name + "'. Expected one of the following identifiers: " + all_types_str; throw Exception(msg, ErrorCodes::SYNTAX_ERROR); }); } - bool parseLimit(IParserBase::Pos & pos, Expected & expected, bool first, ResourceType & resource_type, ResourceAmount & max) + + bool parseResourceType(IParserBase::Pos & pos, Expected & expected, ResourceType & resource_type) { return IParserBase::wrapParseImpl(pos, [&] { - if (first) - { - if (!ParserKeyword{"MAX"}.ignore(pos, expected)) - return false; - } - else - { - if (!ParserToken{TokenType::Comma}.ignore(pos, expected)) - return false; - - ParserKeyword{"MAX"}.ignore(pos, expected); - } - - std::optional res_resource_type; for (auto rt : ext::range(Quota::MAX_RESOURCE_TYPE)) { if (ParserKeyword{ResourceTypeInfo::get(rt).keyword.c_str()}.ignore(pos, expected)) { - res_resource_type = rt; - break; + resource_type = rt; + return true; } } - if (!res_resource_type) + + ASTPtr ast; + if (!ParserIdentifier{}.parse(pos, ast, expected)) return false; - ResourceAmount res_max; - ASTPtr max_ast; - if (ParserNumber{}.parse(pos, max_ast, expected)) + String name = getIdentifierName(ast); + for (auto rt : ext::range(Quota::MAX_RESOURCE_TYPE)) { - const Field & max_field = max_ast->as().value; - const auto & type_info = ResourceTypeInfo::get(*res_resource_type); - if (type_info.output_denominator == 1) - res_max = applyVisitor(FieldVisitorConvertToNumber(), max_field); - else - res_max = static_cast( - applyVisitor(FieldVisitorConvertToNumber(), max_field) * type_info.output_denominator); + if (ResourceTypeInfo::get(rt).name == name) + { + resource_type = rt; + return true; + } + } + + return false; + }); + } + + + bool parseMaxAmount(IParserBase::Pos & pos, Expected & expected, ResourceType resource_type, ResourceAmount & max) + { + ASTPtr ast; + if (!ParserNumber{}.parse(pos, ast, expected)) + return false; + + const Field & max_field = ast->as().value; + const auto & type_info = ResourceTypeInfo::get(resource_type); + if (type_info.output_denominator == 1) + max = applyVisitor(FieldVisitorConvertToNumber(), max_field); + else + max = static_cast( + applyVisitor(FieldVisitorConvertToNumber(), max_field) * type_info.output_denominator); + return true; + } + + + bool parseLimit(IParserBase::Pos & pos, Expected & expected, bool first, bool & max_prefix_encountered, ResourceType & resource_type, ResourceAmount & max) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (!first && !ParserToken{TokenType::Comma}.ignore(pos, expected)) + return false; + + max_prefix_encountered |= ParserKeyword{"MAX"}.ignore(pos, expected); + + ResourceType res_resource_type; + if (!parseResourceType(pos, expected, res_resource_type)) + return false; + + if (max_prefix_encountered) + { + ParserToken{TokenType::Equals}.ignore(pos, expected); } else + { + if (!ParserKeyword{"MAX"}.ignore(pos, expected)) + return false; + } + + ResourceAmount res_max; + if (!parseMaxAmount(pos, expected, res_resource_type, res_max)) return false; - resource_type = *res_resource_type; + resource_type = res_resource_type; max = res_max; return true; }); } - bool parseLimits(IParserBase::Pos & pos, Expected & expected, ASTCreateQuotaQuery::Limits & limits) + bool parseIntervalWithLimits(IParserBase::Pos & pos, Expected & expected, ASTCreateQuotaQuery::Limits & limits) { return IParserBase::wrapParseImpl(pos, [&] { @@ -126,8 +169,7 @@ namespace new_limits.randomize_interval = ParserKeyword{"RANDOMIZED"}.ignore(pos, expected); - if (!ParserKeyword{"INTERVAL"}.ignore(pos, expected)) - return false; + ParserKeyword{"INTERVAL"}.ignore(pos, expected); ASTPtr num_intervals_ast; if (!ParserNumber{}.parse(pos, num_intervals_ast, expected)) @@ -152,11 +194,12 @@ namespace { ResourceType resource_type; ResourceAmount max; - if (!parseLimit(pos, expected, true, resource_type, max)) + bool max_prefix_encountered = false; + if (!parseLimit(pos, expected, true, max_prefix_encountered, resource_type, max)) return false; new_limits.max[resource_type] = max; - while (parseLimit(pos, expected, false, resource_type, max)) + while (parseLimit(pos, expected, false, max_prefix_encountered, resource_type, max)) new_limits.max[resource_type] = max; } @@ -165,7 +208,7 @@ namespace }); } - bool parseAllLimits(IParserBase::Pos & pos, Expected & expected, std::vector & all_limits) + bool parseIntervalsWithLimits(IParserBase::Pos & pos, Expected & expected, std::vector & all_limits) { return IParserBase::wrapParseImpl(pos, [&] { @@ -173,7 +216,7 @@ namespace do { ASTCreateQuotaQuery::Limits limits; - if (!parseLimits(pos, expected, limits)) + if (!parseIntervalWithLimits(pos, expected, limits)) { all_limits.resize(old_size); return false; @@ -185,6 +228,7 @@ namespace }); } + bool parseToRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) { return IParserBase::wrapParseImpl(pos, [&] @@ -192,7 +236,7 @@ namespace ASTPtr node; ParserRolesOrUsersSet roles_p; roles_p.allowAll().allowRoleNames().allowUserNames().allowCurrentUser().useIDMode(id_mode); - if (roles || !ParserKeyword{"TO"}.ignore(pos, expected) || !roles_p.parse(pos, node, expected)) + if (!ParserKeyword{"TO"}.ignore(pos, expected) || !roles_p.parse(pos, node, expected)) return false; roles = std::static_pointer_cast(node); @@ -256,10 +300,17 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe if (alter && new_name.empty() && (names.size() == 1) && parseRenameTo(pos, expected, new_name)) continue; - if (!key_type && parseKeyType(pos, expected, key_type)) - continue; + if (!key_type) + { + KeyType new_key_type; + if (parseKeyType(pos, expected, new_key_type)) + { + key_type = new_key_type; + continue; + } + } - if (parseAllLimits(pos, expected, all_limits)) + if (parseIntervalsWithLimits(pos, expected, all_limits)) continue; if (cluster.empty() && parseOnCluster(pos, expected, cluster)) diff --git a/src/Parsers/ParserCreateQuotaQuery.h b/src/Parsers/ParserCreateQuotaQuery.h index e2185d4d5ff..03021985357 100644 --- a/src/Parsers/ParserCreateQuotaQuery.h +++ b/src/Parsers/ParserCreateQuotaQuery.h @@ -7,17 +7,17 @@ namespace DB { /** Parses queries like * CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name - * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] - * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} - * {MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number} [,...] | + * [KEYED BY {none | user_name | ip_address | client_key | client_key, user_name | client_key, ip_address} | NOT KEYED] + * [FOR [RANDOMIZED] INTERVAL number {second | minute | hour | day} + * {MAX {{queries | errors | result_rows | result_bytes | read_rows | read_bytes | execution_time} = number} [,...] | * NO LIMITS | TRACKING ONLY} [,...]] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] * * ALTER QUOTA [IF EXISTS] name * [RENAME TO new_name] - * [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] - * [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} - * {MAX {{QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number} } [,...] | + * [KEYED BY {none | user_name | ip_address | client_key | client_key, user_name | client_key, ip_address} | NOT KEYED] + * [FOR [RANDOMIZED] INTERVAL number {second | minute | hour | day} + * {MAX {{queries | errors | result_rows | result_bytes | read_rows | read_bytes | execution_time} = number} [,...] | * NO LIMITS | TRACKING ONLY} [,...]] * [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] */ diff --git a/src/Storages/System/StorageSystemQuotas.cpp b/src/Storages/System/StorageSystemQuotas.cpp index 5d8c0be5861..7ba2d32fe68 100644 --- a/src/Storages/System/StorageSystemQuotas.cpp +++ b/src/Storages/System/StorageSystemQuotas.cpp @@ -26,7 +26,11 @@ namespace { DataTypeEnum8::Values enum_values; for (auto key_type : ext::range(KeyType::MAX)) - enum_values.push_back({KeyTypeInfo::get(key_type).name, static_cast(key_type)}); + { + const auto & type_info = KeyTypeInfo::get(key_type); + if ((key_type != KeyType::NONE) && type_info.base_types.empty()) + enum_values.push_back({type_info.name, static_cast(key_type)}); + } return enum_values; } } @@ -37,8 +41,8 @@ NamesAndTypesList StorageSystemQuotas::getNamesAndTypes() NamesAndTypesList names_and_types{ {"name", std::make_shared()}, {"id", std::make_shared()}, - {"source", std::make_shared()}, - {"key_type", std::make_shared(getKeyTypeEnumValues())}, + {"storage", std::make_shared()}, + {"keys", std::make_shared(std::make_shared(getKeyTypeEnumValues()))}, {"durations", std::make_shared(std::make_shared())}, {"apply_to_all", std::make_shared()}, {"apply_to_list", std::make_shared(std::make_shared())}, @@ -58,7 +62,8 @@ void StorageSystemQuotas::fillData(MutableColumns & res_columns, const Context & auto & column_name = assert_cast(*res_columns[column_index++]); auto & column_id = assert_cast(*res_columns[column_index++]).getData(); auto & column_storage = assert_cast(*res_columns[column_index++]); - auto & column_key_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_key_types = assert_cast(assert_cast(*res_columns[column_index]).getData()).getData(); + auto & column_key_types_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_durations = assert_cast(assert_cast(*res_columns[column_index]).getData()).getData(); auto & column_durations_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_apply_to_all = assert_cast(*res_columns[column_index++]).getData(); @@ -77,7 +82,16 @@ void StorageSystemQuotas::fillData(MutableColumns & res_columns, const Context & column_name.insertData(name.data(), name.length()); column_id.push_back(id); column_storage.insertData(storage_name.data(), storage_name.length()); - column_key_type.push_back(static_cast(key_type)); + + if (key_type != KeyType::NONE) + { + const auto & type_info = KeyTypeInfo::get(key_type); + for (auto base_type : type_info.base_types) + column_key_types.push_back(static_cast(base_type)); + if (type_info.base_types.empty()) + column_key_types.push_back(static_cast(key_type)); + } + column_key_types_offsets.push_back(column_key_types.size()); for (const auto & limits : all_limits) column_durations.push_back(std::chrono::duration_cast(limits.duration).count()); diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index a47af5ad5b8..dab6758cbd6 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -41,7 +41,7 @@ def test_create(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n" assert instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n" - assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q KEYED BY \\'none\\' FOR INTERVAL 1 HOUR MAX QUERIES 100 TO ALL EXCEPT rx\n" + assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 100 TO ALL EXCEPT rx\n" assert instance.query("SHOW GRANTS FOR u1") == "" assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx TO u2\n" assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s1\n" diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py index 6e00bf7241b..ab8077030e6 100644 --- a/tests/integration/test_quota/test.py +++ b/tests/integration/test_quota/test.py @@ -61,7 +61,7 @@ def reset_quotas_and_usage_info(): def test_quota_from_users_xml(): - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] @@ -76,7 +76,7 @@ def test_quota_from_users_xml(): def test_simpliest_quota(): # Simpliest quota doesn't even track usage. copy_quota_xml('simpliest.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[]", 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[]", 0, "['default']", "[]"]] assert system_quota_limits() == "" assert system_quota_usage() == [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] @@ -87,7 +87,7 @@ def test_simpliest_quota(): def test_tracking_quota(): # Now we're tracking usage. copy_quota_xml('tracking.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] assert system_quota_usage() == [["myQuota", "default", 31556952, 0, "\N", 0, "\N", 0, "\N", 0, "\N", 0, "\N", 0, "\N", "\N"]] @@ -101,7 +101,7 @@ def test_tracking_quota(): def test_exceed_quota(): # Change quota, now the limits are tiny so we will exceed the quota. copy_quota_xml('tiny_limits.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1, 1, 1, "\N", 1, "\N", "\N"]] assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1, 0, 1, 0, 1, 0, "\N", 0, 1, 0, "\N", "\N"]] @@ -110,7 +110,7 @@ def test_exceed_quota(): # Change quota, now the limits are enough to execute queries. copy_quota_xml('normal_limits.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1000, 1, "\N", 0, "\N", 0, "\N", 50, 1000, 0, "\N", "\N"]] @@ -119,13 +119,13 @@ def test_exceed_quota(): def test_add_remove_interval(): - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] # Add interval. copy_quota_xml('two_intervals.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952,63113904]", 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952,63113904]", 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], ["myQuota", 63113904, 1, "\N", "\N", "\N", 30000, "\N", 20000, 120]] assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"], @@ -137,7 +137,7 @@ def test_add_remove_interval(): # Remove interval. copy_quota_xml('normal_limits.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"]] @@ -146,7 +146,7 @@ def test_add_remove_interval(): # Remove all intervals. copy_quota_xml('simpliest.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[]", 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[]", 0, "['default']", "[]"]] assert system_quota_limits() == "" assert system_quota_usage() == [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] @@ -155,20 +155,21 @@ def test_add_remove_interval(): # Add one interval back. copy_quota_xml('normal_limits.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] def test_add_remove_quota(): - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] # Add quota. copy_quota_xml('two_quotas.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"], - ["myQuota2", "4590510c-4d13-bf21-ec8a-c2187b092e73", "users.xml", "client key or user name", "[3600,2629746]", 0, "[]", "[]"]] + print system_quotas() + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"], + ["myQuota2", "4590510c-4d13-bf21-ec8a-c2187b092e73", "users.xml", "['client_key','user_name']", "[3600,2629746]", 0, "[]", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], ["myQuota2", 3600, 1, "\N", "\N", 4000, 400000, 4000, 400000, 60], ["myQuota2", 2629746, 0, "\N", "\N", "\N", "\N", "\N", "\N", 1800]] @@ -176,7 +177,7 @@ def test_add_remove_quota(): # Drop quota. copy_quota_xml('normal_limits.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] @@ -188,25 +189,25 @@ def test_add_remove_quota(): # Add one quota back. copy_quota_xml('normal_limits.xml') - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] def test_reload_users_xml_by_timer(): - assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] time.sleep(1) # The modification time of the 'quota.xml' file should be different, # because config files are reload by timer only when the modification time is changed. copy_quota_xml('tiny_limits.xml', reload_immediately=False) - assert_eq_with_retry(instance, "SELECT * FROM system.quotas", [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]]) + assert_eq_with_retry(instance, "SELECT * FROM system.quotas", [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", ['user_name'], "[31556952]", 0, "['default']", "[]"]]) assert_eq_with_retry(instance, "SELECT * FROM system.quota_limits", [["myQuota", 31556952, 0, 1, 1, 1, "\N", 1, "\N", "\N"]]) def test_dcl_introspection(): assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000 TO default\n" + assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t0\\t1000\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t1000\\t0\\t\\\\N\\t.*\\t\\\\N\n", instance.query("SHOW QUOTA")) @@ -217,7 +218,7 @@ def test_dcl_introspection(): # Add interval. copy_quota_xml('two_intervals.xml') assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000, FOR RANDOMIZED INTERVAL 2 YEAR MAX RESULT BYTES 30000, READ BYTES 20000, EXECUTION TIME 120 TO default\n" + assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000, FOR RANDOMIZED INTERVAL 2 year MAX result_bytes = 30000, read_bytes = 20000, execution_time = 120 TO default\n" assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n" "myQuota\\tdefault\\t.*\\t63113904\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t30000\\t0\\t\\\\N\\t0\\t20000\\t.*\\t120", instance.query("SHOW QUOTA")) @@ -225,8 +226,8 @@ def test_dcl_introspection(): # Drop interval, add quota. copy_quota_xml('two_quotas.xml') assert instance.query("SHOW QUOTAS") == "myQuota\nmyQuota2\n" - assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000 TO default\n" - assert instance.query("SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY \\'client key or user name\\' FOR RANDOMIZED INTERVAL 1 HOUR MAX RESULT ROWS 4000, RESULT BYTES 400000, READ ROWS 4000, READ BYTES 400000, EXECUTION TIME 60, FOR INTERVAL 1 MONTH MAX EXECUTION TIME 1800\n" + assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" + assert instance.query("SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", instance.query("SHOW QUOTA")) @@ -242,7 +243,7 @@ def test_dcl_management(): assert instance.query("SHOW QUOTA") == "" instance.query("CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER") - assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 5 QUARTER MAX QUERIES 123 TO default\n" + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA FOR INTERVAL 5 quarter MAX queries = 123 TO default\n" assert re.match("qA\\t\\t.*\\t39446190\\t0\\t123\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", instance.query("SHOW QUOTA")) @@ -251,7 +252,7 @@ def test_dcl_management(): instance.query("SHOW QUOTA")) instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 321, MAX ERRORS 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME 0.5") - assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 30 MINUTE MAX EXECUTION TIME 0.5, FOR INTERVAL 5 QUARTER MAX QUERIES 321, ERRORS 10 TO default\n" + assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default\n" assert re.match("qA\\t\\t.*\\t1800\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t0.5\n" "qA\\t\\t.*\\t39446190\\t1\\t321\\t0\\t10\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", instance.query("SHOW QUOTA")) @@ -270,7 +271,7 @@ def test_dcl_management(): instance.query("SHOW QUOTA")) instance.query("ALTER QUOTA qA RENAME TO qB") - assert instance.query("SHOW CREATE QUOTA qB") == "CREATE QUOTA qB KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY TO default\n" + assert instance.query("SHOW CREATE QUOTA qB") == "CREATE QUOTA qB FOR RANDOMIZED INTERVAL 16 month TRACKING ONLY TO default\n" assert re.match("qB\\t\\t.*\\t42075936\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", instance.query("SHOW QUOTA")) diff --git a/tests/queries/0_stateless/01033_quota_dcl.reference b/tests/queries/0_stateless/01033_quota_dcl.reference index 7bd2d2923d2..2ad3c3ad784 100644 --- a/tests/queries/0_stateless/01033_quota_dcl.reference +++ b/tests/queries/0_stateless/01033_quota_dcl.reference @@ -1,2 +1,2 @@ default -CREATE QUOTA default KEYED BY \'user name\' FOR INTERVAL 1 HOUR TRACKING ONLY TO default, readonly +CREATE QUOTA default KEYED BY user_name FOR INTERVAL 1 hour TRACKING ONLY TO default, readonly From a9599d0a37c4e28ef853963901c6d09c8e6d52e1 Mon Sep 17 00:00:00 2001 From: Tom Bombadil <565258751@qq.com> Date: Tue, 16 Jun 2020 02:40:20 +0800 Subject: [PATCH 228/329] Update index.md (#11674) * Update index.md optimize chinese-doc translation * Update index.md * Update index.md Co-authored-by: Ivan Blinkov --- docs/zh/sql-reference/index.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/zh/sql-reference/index.md b/docs/zh/sql-reference/index.md index aed96c4b34f..c47c20b9cf9 100644 --- a/docs/zh/sql-reference/index.md +++ b/docs/zh/sql-reference/index.md @@ -1,15 +1,13 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "SQL\u53C2\u8003" +toc_folder_title: SQL参考 toc_hidden: true toc_priority: 28 -toc_title: "\u9690\u85CF" +toc_title: hidden --- # SQL参考 {#sql-reference} -ClickHouse支持以下类型的查询: +ClickHouse支持以下形式的查询: - [SELECT](statements/select/index.md) - [INSERT INTO](statements/insert-into.md) @@ -17,4 +15,4 @@ ClickHouse支持以下类型的查询: - [ALTER](statements/alter.md#query_language_queries_alter) - [其他类型的查询](statements/misc.md) -[原始文章](https://clickhouse.tech/docs/en/sql-reference/) +[原始文档](https://clickhouse.tech/docs/zh/sql-reference/) From 3f8d72c3724b2d80d698aaef643ae036831118a7 Mon Sep 17 00:00:00 2001 From: Yuntao Wu Date: Tue, 16 Jun 2020 02:41:31 +0800 Subject: [PATCH 229/329] =?UTF-8?q?merge=20translates=20into=20"=E5=90=88?= =?UTF-8?q?=E5=B9=B6"=20better=20(#11659)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * merge translates into "合并" better * Update index.md Co-authored-by: Ivan Blinkov --- docs/zh/engines/table-engines/mergetree-family/index.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/index.md b/docs/zh/engines/table-engines/mergetree-family/index.md index 746d9f03281..c24dd02bb72 100644 --- a/docs/zh/engines/table-engines/mergetree-family/index.md +++ b/docs/zh/engines/table-engines/mergetree-family/index.md @@ -1,7 +1,5 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_folder_title: "\u6885\u6811\u5BB6\u65CF" +toc_folder_title: "合并树家族" toc_priority: 28 --- From 4f8c7bcf78483f86c088185147391686c61c30f7 Mon Sep 17 00:00:00 2001 From: Yuntao Wu Date: Tue, 16 Jun 2020 02:42:35 +0800 Subject: [PATCH 230/329] update some errors (#11656) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit collapse means “折叠” in Chinese engine means “引擎” in Chinese when we are developing --- .../mergetree-family/versionedcollapsingmergetree.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index 19caae5e1a1..3ee35e7cdf7 100644 --- a/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -33,7 +33,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] 有关查询参数的说明，请参阅 [查询说明](../../../sql-reference/statements/create.md). -**发动机参数** +**引擎参数** ``` sql VersionedCollapsingMergeTree(sign, version) @@ -79,7 +79,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -## 崩溃 {#table_engines_versionedcollapsingmergetree} +## 折叠 {#table_engines_versionedcollapsingmergetree} ### 数据 {#data} From 42ff73eb00401ee1609ca65108582dd28f91686c Mon Sep 17 00:00:00 2001 From: bluebirddm Date: Tue, 16 Jun 2020 02:43:06 +0800 Subject: [PATCH 231/329] Update versionedcollapsingmergetree.md (#11654) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update versionedcollapsingmergetree.md 简单翻译 * Update versionedcollapsingmergetree.md Co-authored-by: Ivan Blinkov --- .../versionedcollapsingmergetree.md | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index 3ee35e7cdf7..257bc2ad203 100644 --- a/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -1,6 +1,4 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 37 toc_title: "\u7248\u672C\u96C6\u5408\u5728\u65B0\u6811" --- @@ -39,17 +37,17 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] VersionedCollapsingMergeTree(sign, version) ``` -- `sign` — Name of the column with the type of row: `1` 是一个 “state” 行, `-1` 是一个 “cancel” 划 +- `sign` — 指定行类型的列名: `1` 是一个 “state” 行, `-1` 是一个 “cancel” 划 列数据类型应为 `Int8`. -- `version` — Name of the column with the version of the object state. +- `version` — 指定对象状态版本的列名。 列数据类型应为 `UInt*`. -**查询子句** +**查询 Clauses** -当创建一个 `VersionedCollapsingMergeTree` 表，相同 [条款](mergetree.md) 需要创建一个时 `MergeTree` 桌子 +当创建一个 `VersionedCollapsingMergeTree` 表时，跟创建一个 `MergeTree`表的时候需要相同 [Clause](mergetree.md)
@@ -69,11 +67,11 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] 所有的参数，除了 `sign` 和 `version` 具有相同的含义 `MergeTree`. -- `sign` — Name of the column with the type of row: `1` 是一个 “state” 行, `-1` 是一个 “cancel” 划 +- `sign` — 指定行类型的列名: `1` 是一个 “state” 行, `-1` 是一个 “cancel” 划 Column Data Type — `Int8`. -- `version` — Name of the column with the version of the object state. +- `version` — 指定对象状态版本的列名。 列数据类型应为 `UInt*`. @@ -125,23 +123,23 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] 1. 写入数据的程序应该记住对象的状态以取消它。 该 “cancel” 字符串应该是 “state” 与相反的字符串 `Sign`. 这增加了存储的初始大小，但允许快速写入数据。 2. 列中长时间增长的数组由于写入负载而降低了引擎的效率。 数据越简单，效率就越高。 -3. `SELECT` 结果很大程度上取决于对象变化历史的一致性。 准备插入数据时要准确。 您可以通过不一致的数据获得不可预测的结果，例如会话深度等非负指标的负值。 +3. `SELECT` 结果很大程度上取决于对象变化历史的一致性。 准备插入数据时要准确。 不一致的数据将导致不可预测的结果，例如会话深度等非负指标的负值。 ### 算法 {#table_engines-versionedcollapsingmergetree-algorithm} -当ClickHouse合并数据部分时，它会删除具有相同主键和版本且不同主键和版本的每对行 `Sign`. 行的顺序并不重要。 +当ClickHouse合并数据部分时，它会删除具有相同主键和版本但 `Sign`值不同的一对行. 行的顺序并不重要。 当ClickHouse插入数据时，它会按主键对行进行排序。 如果 `Version` 列不在主键中，ClickHouse将其隐式添加到主键作为最后一个字段并使用它进行排序。 ## 选择数据 {#selecting-data} -ClickHouse不保证具有相同主键的所有行都将位于相同的结果数据部分中，甚至位于相同的物理服务器上。 对于写入数据和随后合并数据部分都是如此。 此外，ClickHouse流程 `SELECT` 具有多个线程的查询，并且无法预测结果中的行顺序。 这意味着聚合是必需的，如果有必要得到完全 “collapsed” 从数据 `VersionedCollapsingMergeTree` 桌子 +ClickHouse不保证具有相同主键的所有行都将位于相同的结果数据部分中，甚至位于相同的物理服务器上。 对于写入数据和随后合并数据部分都是如此。 此外，ClickHouse流程 `SELECT` 具有多个线程的查询，并且无法预测结果中的行顺序。 这意味着，如果有必要从`VersionedCollapsingMergeTree` 表中得到完全 “collapsed” 的数据，聚合是必需的。 要完成折叠，请使用 `GROUP BY` 考虑符号的子句和聚合函数。 例如，要计算数量，请使用 `sum(Sign)` 而不是 `count()`. 要计算的东西的总和，使用 `sum(Sign * x)` 而不是 `sum(x)`，并添加 `HAVING sum(Sign) > 0`. 聚合 `count`, `sum` 和 `avg` 可以这样计算。 聚合 `uniq` 如果对象至少具有一个非折叠状态，则可以计算。 聚合 `min` 和 `max` 无法计算是因为 `VersionedCollapsingMergeTree` 不保存折叠状态值的历史记录。 -如果您需要提取数据 “collapsing” 但是，如果没有聚合（例如，要检查是否存在其最新值与某些条件匹配的行），则可以使用 `FINAL` 修饰符 `FROM` 条款 这种方法效率低下，不应与大型表一起使用。 +如果您需要提取数据 “collapsing” 但是，如果没有聚合（例如，要检查是否存在其最新值与某些条件匹配的行），则可以使用 `FINAL` 修饰 `FROM` 条件这种方法效率低下，不应与大型表一起使用。 ## 使用示例 {#example-of-use} @@ -233,6 +231,6 @@ SELECT * FROM UAct FINAL └─────────────────────┴───────────┴──────────┴──────┴─────────┘ ``` -这是一个非常低效的方式来选择数据。 不要把它用于大桌子。 +这是一个非常低效的方式来选择数据。 不要把它用于数据量大的表。 [原始文章](https://clickhouse.tech/docs/en/operations/table_engines/versionedcollapsingmergetree/) From 1474dd6d690c97a185dab462689e28ad0fb0ff93 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 15 Jun 2020 21:44:05 +0300 Subject: [PATCH 232/329] DOCS-646: randomString (#11610) * [CLICKHOUSEDOCS] Document the "randomString" function (#121) * Add description of randomString function * Add description for randomString * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Changed example * Add russian version * Fixed links * Fixed links Co-authored-by: Anna Devyatova Co-authored-by: BayoNet * CLICKHOUSEDOCS-646: Updated text. Fixed links. * CLICKHOUSEDOCS-646: Fixed more links. Co-authored-by: Sergei Shtykov Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> Co-authored-by: Anna Devyatova --- .../functions/other-functions.md | 48 +++++++++++++++++++ .../settings.md | 4 +- .../functions/other-functions.md | 48 +++++++++++++++++++ docs/ru/sql-reference/statements/system.md | 2 +- .../sql-reference/table-functions/generate.md | 2 +- 5 files changed, 100 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 9aa26f32b18..18641614bef 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -1200,4 +1200,52 @@ SELECT number, randomPrintableASCII(30) as str, length(str) FROM system.numbers └────────┴────────────────────────────────┴──────────────────────────────────┘ ``` +## randomString {#randomstring} + +Generates a binary string of the specified length filled with random bytes (including zero bytes). + +**Syntax** + +``` sql +randomString(length) +``` + +**Parameters** + +- `length` — String length. Positive integer. + +**Returned value** + +- String filled with random bytes. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +``` sql +SELECT randomString(30) AS str, length(str) AS len FROM numbers(2) FORMAT Vertical; +``` + +Result: + +``` text +Row 1: +────── +str: 3 G : pT ?w тi k aV f6 +len: 30 + +Row 2: +────── +str: 9 ,] ^ ) ]?? 8 +len: 30 +``` + +**See Also** + +- [generateRandom](../../sql-reference/table-functions/generate.md#generaterandom) +- [randomPrintableASCII](../../sql-reference/functions/other-functions.md#randomascii) + + [Original article](https://clickhouse.tech/docs/en/query_language/functions/other_functions/) diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index e3c1629a46a..5bfedf4c520 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -78,7 +78,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat default ``` -## dictionaries\_config {#dictionaries-config} +## dictionaries\_config {#server_configuration_parameters-dictionaries_config} Путь к конфигурации внешних словарей. @@ -95,7 +95,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat *_dictionary.xml ``` -## dictionaries\_lazy\_load {#dictionaries-lazy-load} +## dictionaries\_lazy\_load {#server_configuration_parameters-dictionaries_lazy_load} Отложенная загрузка словарей. diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 2c715cd15a5..7161b1a2468 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -1153,4 +1153,52 @@ SELECT number, randomPrintableASCII(30) as str, length(str) FROM system.numbers └────────┴────────────────────────────────┴──────────────────────────────────┘ ``` +## randomString {#randomstring} + +Генерирует бинарную строку заданной длины, заполненную случайными байтами (в том числе нулевыми). + +**Синтаксис** + +``` sql +randomString(length) +``` + +**Параметры** + +- `length` — длина строки. Положительное целое число. + +**Возвращаемое значение** + +- Строка, заполненная случайными байтами. + +Type: [String](../../sql-reference/data-types/string.md). + +**Пример** + +Запрос: + +``` sql +SELECT randomString(30) AS str, length(str) AS len FROM numbers(2) FORMAT Vertical; +``` + +Ответ: + +``` text +Row 1: +────── +str: 3 G : pT ?w тi k aV f6 +len: 30 + +Row 2: +────── +str: 9 ,] ^ ) ]?? 8 +len: 30 +``` + +**Смотрите также** + +- [generateRandom](../../sql-reference/table-functions/generate.md#generaterandom) +- [randomPrintableASCII](../../sql-reference/functions/other-functions.md#randomascii) + + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/other_functions/) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 1b66fa039d9..9a6dccd7d89 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -38,7 +38,7 @@ ## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries} Перегружает все словари, которые были успешно загружены до этого. -По умолчанию включена ленивая загрузка [dictionaries\_lazy\_load](../../sql-reference/statements/system.md#dictionaries-lazy-load), поэтому словари не загружаются автоматически при старте, а только при первом обращении через dictGet или SELECT к ENGINE=Dictionary. После этого такие словари (LOADED) будут перегружаться командой `system reload dictionaries`. +По умолчанию включена ленивая загрузка [dictionaries\_lazy\_load](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load), поэтому словари не загружаются автоматически при старте, а только при первом обращении через dictGet или SELECT к ENGINE=Dictionary. После этого такие словари (LOADED) будут перегружаться командой `system reload dictionaries`. Всегда возвращает `Ok.`, вне зависимости от результата обновления словарей. ## RELOAD DICTIONARY Dictionary\_name {#query_language-system-reload-dictionary} diff --git a/docs/ru/sql-reference/table-functions/generate.md b/docs/ru/sql-reference/table-functions/generate.md index b1abdbf1d63..9e6d36b2a4b 100644 --- a/docs/ru/sql-reference/table-functions/generate.md +++ b/docs/ru/sql-reference/table-functions/generate.md @@ -1,4 +1,4 @@ -# generateRandom {#generateRandom} +# generateRandom {#generaterandom} Генерирует случайные данные с заданной схемой. Позволяет заполнять тестовые таблицы данными. From 7cc54fd4f1f447abf7e8309d45a27fcc7aa547ca Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 15 Jun 2020 21:53:54 +0300 Subject: [PATCH 233/329] renames + perf test --- .../AggregateFunctionSumMap.h | 26 +++++++++---------- tests/performance/sum_map.xml | 2 ++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index 0c4b407b8a8..f9985a3c603 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -28,7 +28,7 @@ namespace ErrorCodes } template -struct AggregateFunctionXxxMapData +struct AggregateFunctionMapData { // Map needs to be ordered to maintain function properties std::map merged_maps; @@ -54,18 +54,18 @@ struct AggregateFunctionXxxMapData */ template -class AggregateFunctionMapOpBase : public IAggregateFunctionDataHelper< - AggregateFunctionXxxMapData>, Derived> +class AggregateFunctionMapBase : public IAggregateFunctionDataHelper< + AggregateFunctionMapData>, Derived> { private: DataTypePtr keys_type; DataTypes values_types; public: - AggregateFunctionMapOpBase( + AggregateFunctionMapBase( const DataTypePtr & keys_type_, const DataTypes & values_types_, const DataTypes & argument_types_, const Array & params_) - : IAggregateFunctionDataHelper>, Derived>(argument_types_, params_) + : IAggregateFunctionDataHelper>, Derived>(argument_types_, params_) , keys_type(keys_type_), values_types(values_types_) {} DataTypePtr getReturnType() const override @@ -305,11 +305,11 @@ public: template class AggregateFunctionSumMap final : - public AggregateFunctionMapOpBase, FieldVisitorSum, overflow, tuple_argument> + public AggregateFunctionMapBase, FieldVisitorSum, overflow, tuple_argument> { private: using Self = AggregateFunctionSumMap; - using Base = AggregateFunctionMapOpBase; + using Base = AggregateFunctionMapBase; public: AggregateFunctionSumMap(const DataTypePtr & keys_type_, DataTypes & values_types_, const DataTypes & argument_types_) @@ -323,7 +323,7 @@ public: template class AggregateFunctionSumMapFiltered final : - public AggregateFunctionMapOpBase, FieldVisitorSum, overflow, @@ -331,7 +331,7 @@ class AggregateFunctionSumMapFiltered final : { private: using Self = AggregateFunctionSumMapFiltered; - using Base = AggregateFunctionMapOpBase; + using Base = AggregateFunctionMapBase; std::unordered_set keys_to_keep; @@ -355,11 +355,11 @@ public: template class AggregateFunctionMinMap final : - public AggregateFunctionMapOpBase, FieldVisitorMin, true, tuple_argument> + public AggregateFunctionMapBase, FieldVisitorMin, true, tuple_argument> { private: using Self = AggregateFunctionMinMap; - using Base = AggregateFunctionMapOpBase; + using Base = AggregateFunctionMapBase; public: AggregateFunctionMinMap(const DataTypePtr & keys_type_, DataTypes & values_types_, const DataTypes & argument_types_) @@ -373,11 +373,11 @@ public: template class AggregateFunctionMaxMap final : - public AggregateFunctionMapOpBase, FieldVisitorMax, true, tuple_argument> + public AggregateFunctionMapBase, FieldVisitorMax, true, tuple_argument> { private: using Self = AggregateFunctionMaxMap; - using Base = AggregateFunctionMapOpBase; + using Base = AggregateFunctionMapBase; public: AggregateFunctionMaxMap(const DataTypePtr & keys_type_, DataTypes & values_types_, const DataTypes & argument_types_) diff --git a/tests/performance/sum_map.xml b/tests/performance/sum_map.xml index cb1a4cb5bc6..cfad530652c 100644 --- a/tests/performance/sum_map.xml +++ b/tests/performance/sum_map.xml @@ -13,6 +13,8 @@ func + minMap + maxMap sumMap sumMapWithOverflow From f0eafed5206e0cbe8f89fd078c733a5889f6b0af Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 15 Jun 2020 21:55:04 +0300 Subject: [PATCH 234/329] DOCS-678: Updated ASOF Join description (#11676) * CLICKHOUSEDOCS-678: Updated ASOF Join Usage. * CLICKHOUSEDOCS-678: Updated templates. * Update docs/ru/sql-reference/statements/select/join.md Co-authored-by: Ivan Blinkov * CLICKHOUSEDOCS-678: Update by comments. Co-authored-by: Sergei Shtykov Co-authored-by: emironyuk Co-authored-by: Ivan Blinkov --- docs/_description_templates/template-function.md | 2 +- docs/_description_templates/template-setting.md | 2 +- docs/en/sql-reference/statements/select/join.md | 6 +++++- docs/ru/sql-reference/statements/select/join.md | 6 +++++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/_description_templates/template-function.md b/docs/_description_templates/template-function.md index 1acf92cb501..b69d7ed5309 100644 --- a/docs/_description_templates/template-function.md +++ b/docs/_description_templates/template-function.md @@ -1,4 +1,4 @@ -## function-name {#function-name-in-lower-case} +## functionName {#functionname-in-lower-case} Short description. diff --git a/docs/_description_templates/template-setting.md b/docs/_description_templates/template-setting.md index 5a33716f899..fc912aba3e1 100644 --- a/docs/_description_templates/template-setting.md +++ b/docs/_description_templates/template-setting.md @@ -1,4 +1,4 @@ -## setting-name {#setting-name-in-lower-case} +## setting_name {#setting_name} Description. diff --git a/docs/en/sql-reference/statements/select/join.md b/docs/en/sql-reference/statements/select/join.md index 5ac3f4a0e25..87bc542dbdc 100644 --- a/docs/en/sql-reference/statements/select/join.md +++ b/docs/en/sql-reference/statements/select/join.md @@ -51,7 +51,11 @@ Modifies how matching by "join keys" is performed `ASOF JOIN` is useful when you need to join records that have no exact match. -Tables for `ASOF JOIN` must have an ordered sequence column. This column cannot be alone in a table, and should be one of the data types: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date`, and `DateTime`. +Algorithm requires the special column in tables. This column: + +- Must contain an ordered sequence. +- Can be one of the following types: [Int*, UInt*](../../data-types/int-uint.md), [Float*](../../data-types/float.md), [Date](../../data-types/date.md), [DateTime](../../data-types/datetime.md), [Decimal*](../../data-types/decimal.md). +- Can't be the only column in the `JOIN` clause. Syntax `ASOF JOIN ... ON`: diff --git a/docs/ru/sql-reference/statements/select/join.md b/docs/ru/sql-reference/statements/select/join.md index 60f391d888b..26e7ae8257e 100644 --- a/docs/ru/sql-reference/statements/select/join.md +++ b/docs/ru/sql-reference/statements/select/join.md @@ -45,7 +45,11 @@ FROM `ASOF JOIN` применим в том случае, когда необходимо объединять записи, которые не имеют точного совпадения. -Таблицы для `ASOF JOIN` должны иметь столбец с отсортированной последовательностью. Этот столбец не может быть единственным в таблице и должен быть одного из типов: `UInt32`, `UInt64`, `Float32`, `Float64`, `Date` и `DateTime`. +Для работы алгоритма необходим специальный столбец в таблицах. Этот столбец: + +- Должен содержать упорядоченную последовательность. +- Может быть одного из следующих типов: [Int*, UInt*](../../data-types/int-uint.md), [Float*](../../data-types/float.md), [Date](../../data-types/date.md), [DateTime](../../data-types/datetime.md), [Decimal*](../../data-types/decimal.md). +- Не может быть единственным столбцом в секции `JOIN`. Синтаксис `ASOF JOIN ... ON`: From d6e69211b1ba074f5e51fd467e51b892a7e091a6 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 15 Jun 2020 22:05:36 +0300 Subject: [PATCH 235/329] fix test --- tests/queries/0_stateless/01269_create_with_null.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01269_create_with_null.reference b/tests/queries/0_stateless/01269_create_with_null.reference index 739063af67f..e4945eed114 100644 --- a/tests/queries/0_stateless/01269_create_with_null.reference +++ b/tests/queries/0_stateless/01269_create_with_null.reference @@ -1,4 +1,4 @@ Nullable(Int32) Int32 Nullable(Int32) Int32 -CREATE TABLE default.data_null\n(\n `a` Nullable(Int32), \n `b` Int32, \n `c` Nullable(Int32), \n `d` Int32\n)\nENGINE = Memory() +CREATE TABLE default.data_null\n(\n `a` Nullable(Int32),\n `b` Int32,\n `c` Nullable(Int32),\n `d` Int32\n)\nENGINE = Memory() Nullable(Int32) Int32 Nullable(Int32) Nullable(Int32) -CREATE TABLE default.set_null\n(\n `a` Nullable(Int32), \n `b` Int32, \n `c` Nullable(Int32), \n `d` Nullable(Int32)\n)\nENGINE = Memory() +CREATE TABLE default.set_null\n(\n `a` Nullable(Int32),\n `b` Int32,\n `c` Nullable(Int32),\n `d` Nullable(Int32)\n)\nENGINE = Memory() From 15132d47c9b1f4c7a0ef7a41dfccc81e53c11b57 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 5 Jun 2020 21:56:33 +0300 Subject: [PATCH 236/329] Slightly improve syntax of CREATE POLICY. Also rename column source=>storage in table system.row_policies. --- .../InterpreterCreateRowPolicyQuery.cpp | 8 +- ...InterpreterShowCreateAccessEntityQuery.cpp | 2 +- src/Parsers/ASTCreateRowPolicyQuery.cpp | 39 ++--- src/Parsers/ASTCreateRowPolicyQuery.h | 7 +- src/Parsers/ParserCreateRowPolicyQuery.cpp | 128 ++++++++------ src/Parsers/ParserCreateRowPolicyQuery.h | 4 +- src/Parsers/ParserRowPolicyName.cpp | 163 +++++++++++------- .../System/StorageSystemRowPolicies.cpp | 2 +- 8 files changed, 202 insertions(+), 151 deletions(-) diff --git a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp index 3a0ee3f16a1..9dacc9d1bf4 100644 --- a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp +++ b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp @@ -30,12 +30,8 @@ namespace if (query.is_restrictive) policy.setRestrictive(*query.is_restrictive); - for (auto condition_type : ext::range(RowPolicy::MAX_CONDITION_TYPE)) - { - const auto & condition = query.conditions[condition_type]; - if (condition) - policy.conditions[condition_type] = *condition ? serializeAST(**condition) : String{}; - } + for (const auto & [condition_type, condition] : query.conditions) + policy.conditions[condition_type] = condition ? serializeAST(*condition) : String{}; if (override_to_roles) policy.to_roles = *override_to_roles; diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 18a08d21e93..5c4173d7aa3 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -179,7 +179,7 @@ namespace { ParserExpression parser; ASTPtr expr = parseQuery(parser, condition, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - query->conditions[static_cast(type)] = expr; + query->conditions.emplace_back(type, std::move(expr)); } } diff --git a/src/Parsers/ASTCreateRowPolicyQuery.cpp b/src/Parsers/ASTCreateRowPolicyQuery.cpp index 580642f2da5..640b030b6cf 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/ASTCreateRowPolicyQuery.cpp @@ -14,7 +14,6 @@ namespace { using ConditionType = RowPolicy::ConditionType; using ConditionTypeInfo = RowPolicy::ConditionTypeInfo; - constexpr auto MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; void formatRenameTo(const String & new_short_name, const IAST::FormatSettings & settings) @@ -26,21 +25,22 @@ namespace void formatAsRestrictiveOrPermissive(bool is_restrictive, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " AS " << (is_restrictive ? "RESTRICTIVE" : "PERMISSIVE") - << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " AS " << (settings.hilite ? IAST::hilite_none : "") + << (is_restrictive ? "restrictive" : "permissive"); } void formatConditionalExpression(const ASTPtr & expr, const IAST::FormatSettings & settings) { + settings.ostr << " "; if (expr) expr->format(settings); else - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NONE" << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "NONE" << (settings.hilite ? IAST::hilite_none : ""); } - void formatCondition(const boost::container::flat_set & commands, const String & filter, const String & check, bool alter, const IAST::FormatSettings & settings) + void formatForClause(const boost::container::flat_set & commands, const String & filter, const String & check, bool alter, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " FOR " << (settings.hilite ? IAST::hilite_none : ""); bool need_comma = false; @@ -52,27 +52,23 @@ namespace } if (!filter.empty()) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " USING " << (settings.hilite ? IAST::hilite_none : "") << filter; + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " USING" << (settings.hilite ? IAST::hilite_none : "") << filter; if (!check.empty() && (alter || (check != filter))) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH CHECK " << (settings.hilite ? IAST::hilite_none : "") << check; + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH CHECK" << (settings.hilite ? IAST::hilite_none : "") << check; } - void formatMultipleConditions(const std::array, MAX_CONDITION_TYPE> & conditions, bool alter, const IAST::FormatSettings & settings) + void formatForClauses(const std::vector> & conditions, bool alter, const IAST::FormatSettings & settings) { - std::array conditions_as_strings; + std::vector> conditions_as_strings; std::stringstream temp_sstream; IAST::FormatSettings temp_settings(temp_sstream, settings); - for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + for (const auto & [condition_type, condition] : conditions) { - const auto & condition = conditions[condition_type]; - if (condition) - { - formatConditionalExpression(*condition, temp_settings); - conditions_as_strings[condition_type] = temp_sstream.str(); - temp_sstream.str(""); - } + formatConditionalExpression(condition, temp_settings); + conditions_as_strings.emplace_back(condition_type, temp_sstream.str()); + temp_sstream.str(""); } boost::container::flat_set commands; @@ -85,9 +81,8 @@ namespace check.clear(); /// Collect commands using the same filter and check conditions. - for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + for (auto & [condition_type, condition] : conditions_as_strings) { - const String & condition = conditions_as_strings[condition_type]; if (condition.empty()) continue; const auto & type_info = ConditionTypeInfo::get(condition_type); @@ -106,11 +101,11 @@ namespace continue; } commands.emplace(type_info.command); - conditions_as_strings[condition_type].clear(); /// Skip this condition on the next iteration. + condition.clear(); /// Skip this condition on the next iteration. } if (!filter.empty() || !check.empty()) - formatCondition(commands, filter, check, alter, settings); + formatForClause(commands, filter, check, alter, settings); } while (!filter.empty() || !check.empty()); } @@ -167,7 +162,7 @@ void ASTCreateRowPolicyQuery::formatImpl(const FormatSettings & settings, Format if (is_restrictive) formatAsRestrictiveOrPermissive(*is_restrictive, settings); - formatMultipleConditions(conditions, alter, settings); + formatForClauses(conditions, alter, settings); if (roles && (!roles->empty() || alter)) formatToRoles(*roles, settings); diff --git a/src/Parsers/ASTCreateRowPolicyQuery.h b/src/Parsers/ASTCreateRowPolicyQuery.h index 4a7572eaefd..9d0e2fcce7b 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.h +++ b/src/Parsers/ASTCreateRowPolicyQuery.h @@ -3,7 +3,6 @@ #include #include #include -#include #include @@ -13,7 +12,7 @@ class ASTRowPolicyNames; class ASTRolesOrUsersSet; /** CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] name ON [database.]table - * [AS {PERMISSIVE | RESTRICTIVE}] + * [AS {permissive | restrictive}] * [FOR {SELECT | INSERT | UPDATE | DELETE | ALL}] * [USING condition] * [WITH CHECK condition] [,...] @@ -21,7 +20,7 @@ class ASTRolesOrUsersSet; * * ALTER [ROW] POLICY [IF EXISTS] name ON [database.]table * [RENAME TO new_name] - * [AS {PERMISSIVE | RESTRICTIVE}] + * [AS {permissive | restrictive}] * [FOR {SELECT | INSERT | UPDATE | DELETE | ALL}] * [USING {condition | NONE}] * [WITH CHECK {condition | NONE}] [,...] @@ -41,7 +40,7 @@ public: String new_short_name; std::optional is_restrictive; - std::array, RowPolicy::MAX_CONDITION_TYPE> conditions; /// `nullopt` means "not set", `nullptr` means set to NONE. + std::vector> conditions; /// `nullptr` means set to NONE. std::shared_ptr roles; diff --git a/src/Parsers/ParserCreateRowPolicyQuery.cpp b/src/Parsers/ParserCreateRowPolicyQuery.cpp index 061cca4ce63..c9fe15d391f 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/ParserCreateRowPolicyQuery.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace DB @@ -33,7 +34,7 @@ namespace }); } - bool parseAsRestrictiveOrPermissive(IParserBase::Pos & pos, Expected & expected, std::optional & is_restrictive) + bool parseAsRestrictiveOrPermissive(IParserBase::Pos & pos, Expected & expected, bool & is_restrictive) { return IParserBase::wrapParseImpl(pos, [&] { @@ -54,7 +55,7 @@ namespace }); } - bool parseConditionalExpression(IParserBase::Pos & pos, Expected & expected, std::optional & expr) + bool parseConditionalExpression(IParserBase::Pos & pos, Expected & expected, ASTPtr & expr) { return IParserBase::wrapParseImpl(pos, [&] { @@ -74,68 +75,80 @@ namespace }); } - bool parseConditions( - IParserBase::Pos & pos, Expected & expected, bool alter, std::array, MAX_CONDITION_TYPE> & conditions) + void addAllCommands(boost::container::flat_set & commands) + { + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const std::string_view & command = ConditionTypeInfo::get(condition_type).command; + commands.emplace(command); + } + } + + bool parseCommands(IParserBase::Pos & pos, Expected & expected, boost::container::flat_set & commands) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (ParserKeyword{"ALL"}.ignore(pos, expected)) + { + addAllCommands(commands); + return true; + } + + boost::container::flat_set res_commands; + do + { + bool found_keyword = false; + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const std::string_view & command = ConditionTypeInfo::get(condition_type).command; + if (ParserKeyword{command.data()}.ignore(pos, expected)) + { + res_commands.emplace(command); + found_keyword = true; + break; + } + } + + if (!found_keyword) + return false; + } + while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + + commands = std::move(res_commands); + return true; + }); + } + + bool parseForClause(IParserBase::Pos & pos, Expected & expected, bool alter, std::vector> & conditions) { return IParserBase::wrapParseImpl(pos, [&] { boost::container::flat_set commands; - auto add_all_commands = [&] - { - for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) - { - const std::string_view & command = ConditionTypeInfo::get(condition_type).command; - commands.emplace(command); - } - }; - if (ParserKeyword{"FOR"}.ignore(pos, expected)) { - do - { - size_t old_size = commands.size(); - if (ParserKeyword{"ALL"}.ignore(pos, expected)) - { - add_all_commands(); - } - else - { - for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) - { - const std::string_view & command = ConditionTypeInfo::get(condition_type).command; - if (ParserKeyword{command.data()}.ignore(pos, expected)) - { - commands.emplace(command); - break; - } - } - } - if (commands.size() == old_size) - return false; - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + if (!parseCommands(pos, expected, commands)) + return false; } + else + addAllCommands(commands); std::optional filter; std::optional check; if (ParserKeyword{"USING"}.ignore(pos, expected)) { - if (!parseConditionalExpression(pos, expected, filter)) + if (!parseConditionalExpression(pos, expected, filter.emplace())) return false; } if (ParserKeyword{"WITH CHECK"}.ignore(pos, expected)) { - if (!parseConditionalExpression(pos, expected, check)) + if (!parseConditionalExpression(pos, expected, check.emplace())) return false; } if (!filter && !check) return false; - if (commands.empty()) - add_all_commands(); - if (!check && !alter) check = filter; @@ -145,9 +158,9 @@ namespace if (commands.count(type_info.command)) { if (type_info.is_check && check) - conditions[condition_type] = check; + conditions.emplace_back(condition_type, *check); else if (filter) - conditions[condition_type] = filter; + conditions.emplace_back(condition_type, *filter); } } @@ -155,15 +168,15 @@ namespace }); } - bool parseMultipleConditions( - IParserBase::Pos & pos, Expected & expected, bool alter, std::array, MAX_CONDITION_TYPE> & conditions) + bool parseForClauses( + IParserBase::Pos & pos, Expected & expected, bool alter, std::vector> & conditions) { return IParserBase::wrapParseImpl(pos, [&] { - std::array, MAX_CONDITION_TYPE> res_conditions; + std::vector> res_conditions; do { - if (!parseConditions(pos, expected, alter, res_conditions)) + if (!parseForClause(pos, expected, alter, res_conditions)) return false; } while (ParserToken{TokenType::Comma}.ignore(pos, expected)); @@ -178,7 +191,7 @@ namespace return IParserBase::wrapParseImpl(pos, [&] { ASTPtr ast; - if (roles || !ParserKeyword{"TO"}.ignore(pos, expected)) + if (!ParserKeyword{"TO"}.ignore(pos, expected)) return false; ParserRolesOrUsersSet roles_p; @@ -244,18 +257,29 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & String new_short_name; std::optional is_restrictive; - std::array, MAX_CONDITION_TYPE> conditions; + std::vector> conditions; while (true) { if (alter && new_short_name.empty() && (names->name_parts.size() == 1) && parseRenameTo(pos, expected, new_short_name)) continue; - if (!is_restrictive && parseAsRestrictiveOrPermissive(pos, expected, is_restrictive)) - continue; + if (!is_restrictive) + { + bool new_is_restrictive; + if (parseAsRestrictiveOrPermissive(pos, expected, new_is_restrictive)) + { + is_restrictive = new_is_restrictive; + continue; + } + } - if (parseMultipleConditions(pos, expected, alter, conditions)) + std::vector> new_conditions; + if (parseForClauses(pos, expected, alter, new_conditions)) + { + boost::range::push_back(conditions, std::move(new_conditions)); continue; + } if (cluster.empty() && parseOnCluster(pos, expected, cluster)) continue; diff --git a/src/Parsers/ParserCreateRowPolicyQuery.h b/src/Parsers/ParserCreateRowPolicyQuery.h index 1a64f2e893c..f05dca8179c 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.h +++ b/src/Parsers/ParserCreateRowPolicyQuery.h @@ -7,7 +7,7 @@ namespace DB { /** Parses queries like * CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] name ON [database.]table - * [AS {PERMISSIVE | RESTRICTIVE}] + * [AS {permissive | restrictive}] * [FOR {SELECT | INSERT | UPDATE | DELETE | ALL}] * [USING condition] * [WITH CHECK condition] [,...] @@ -15,7 +15,7 @@ namespace DB * * ALTER [ROW] POLICY [IF EXISTS] name ON [database.]table * [RENAME TO new_name] - * [AS {PERMISSIVE | RESTRICTIVE}] + * [AS {permissive | restrictive}] * [FOR {SELECT | INSERT | UPDATE | DELETE | ALL}] * [USING {condition | NONE}] * [WITH CHECK {condition | NONE}] [,...] diff --git a/src/Parsers/ParserRowPolicyName.cpp b/src/Parsers/ParserRowPolicyName.cpp index 8f1ef91f7c1..a74132cdaca 100644 --- a/src/Parsers/ParserRowPolicyName.cpp +++ b/src/Parsers/ParserRowPolicyName.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -19,25 +20,46 @@ namespace } - bool parseOnDatabaseAndTableName(IParser::Pos & pos, Expected & expected, String & database, String & table_name) + bool parseDBAndTableName(IParser::Pos & pos, Expected & expected, String & database, String & table_name) { return IParserBase::wrapParseImpl(pos, [&] { - if (!ParserKeyword{"ON"}.ignore(pos, expected)) + String res_database, res_table_name; + if (!parseDatabaseAndTableName(pos, expected, res_database, res_table_name)) return false; - return parseDatabaseAndTableName(pos, expected, database, table_name); + /// If table is specified without DB it cannot be followed by "ON" + /// (but can be followed by "ON CLUSTER"). + /// The following code is necessary to figure out while parsing something like + /// policy1 ON table1, policy2 ON table2 + /// that policy2 is another policy, not another table. + auto end_pos = pos; + if (res_database.empty() && ParserKeyword{"ON"}.ignore(pos, expected)) + { + String unused; + if (ASTQueryWithOnCluster::parse(pos, unused, expected)) + pos = end_pos; + else + return false; + } + + database = std::move(res_database); + table_name = std::move(res_table_name); + return true; }); } - bool parseOnDatabaseAndTableName(IParser::Pos & pos, Expected & expected, std::pair & database_and_table_name) + bool parseOnDBAndTableName(IParser::Pos & pos, Expected & expected, String & database, String & table_name) { - return parseOnDatabaseAndTableName(pos, expected, database_and_table_name.first, database_and_table_name.second); + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{"ON"}.ignore(pos, expected) && parseDBAndTableName(pos, expected, database, table_name); + }); } - bool parseOnDatabaseAndTableNames(IParser::Pos & pos, Expected & expected, std::vector> & database_and_table_names) + bool parseOnDBAndTableNames(IParser::Pos & pos, Expected & expected, std::vector> & database_and_table_names) { return IParserBase::wrapParseImpl(pos, [&] { @@ -49,23 +71,76 @@ namespace do { String database, table_name; - if (!parseDatabaseAndTableName(pos, expected, database, table_name)) - return false; - - String unused; - if (pos_before_comma && database.empty() && ParserKeyword{"ON"}.ignore(pos, expected) - && !ASTQueryWithOnCluster::parse(pos, unused, expected)) + if (!parseDBAndTableName(pos, expected, database, table_name)) { + if (!pos_before_comma) + return false; pos = *pos_before_comma; break; } - res.emplace_back(std::move(database), std::move(table_name)); pos_before_comma = pos; } while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + database_and_table_names = std::move(res); return true; + }); + } + + + bool parseRowPolicyNamesAroundON(IParser::Pos & pos, Expected & expected, + bool allow_multiple_short_names, + bool allow_multiple_tables, + bool allow_on_cluster, + std::vector & name_parts, + String & cluster) + { + return IParserBase::wrapParseImpl(pos, [&] + { + std::vector short_names; + if (allow_multiple_short_names) + { + if (!parseIdentifiersOrStringLiterals(pos, expected, short_names)) + return false; + } + else + { + if (!parseIdentifierOrStringLiteral(pos, expected, short_names.emplace_back())) + return false; + } + + String res_cluster; + if (allow_on_cluster) + parseOnCluster(pos, expected, res_cluster); + + std::vector> database_and_table_names; + if (allow_multiple_tables && (short_names.size() == 1)) + { + if (!parseOnDBAndTableNames(pos, expected, database_and_table_names)) + return false; + } + else + { + String database, table_name; + if (!parseOnDBAndTableName(pos, expected, database, table_name)) + return false; + database_and_table_names.emplace_back(std::move(database), std::move(table_name)); + } + + + if (allow_on_cluster && res_cluster.empty()) + parseOnCluster(pos, expected, res_cluster); + + assert(!short_names.empty()); + assert(!database_and_table_names.empty()); + name_parts.clear(); + for (const String & short_name : short_names) + for (const auto & [database, table_name] : database_and_table_names) + name_parts.push_back({short_name, database, table_name}); + + cluster = std::move(res_cluster); + return true; }); } } @@ -73,21 +148,14 @@ namespace bool ParserRowPolicyName::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - RowPolicy::NameParts name_parts; - if (!parseIdentifierOrStringLiteral(pos, expected, name_parts.short_name)) - return false; - + std::vector name_parts; String cluster; - parseOnCluster(pos, expected, cluster); - - if (!parseOnDatabaseAndTableName(pos, expected, name_parts.database, name_parts.table_name)) + if (!parseRowPolicyNamesAroundON(pos, expected, false, false, allow_on_cluster, name_parts, cluster)) return false; - if (cluster.empty()) - parseOnCluster(pos, expected, cluster); - + assert(name_parts.size() == 1); auto result = std::make_shared(); - result->name_parts = std::move(name_parts); + result->name_parts = std::move(name_parts.front()); result->cluster = std::move(cluster); node = result; return true; @@ -101,46 +169,15 @@ bool ParserRowPolicyNames::parseImpl(Pos & pos, ASTPtr & node, Expected & expect do { - std::vector short_names; - bool allowed_multiple_short_names = name_parts.empty(); - if (allowed_multiple_short_names) - { - if (!parseIdentifiersOrStringLiterals(pos, expected, short_names)) - return false; - } - else - { - if (!parseIdentifierOrStringLiteral(pos, expected, short_names.emplace_back())) - return false; - } + std::vector new_name_parts; + if (!parseRowPolicyNamesAroundON(pos, expected, name_parts.empty(), name_parts.empty(), allow_on_cluster, new_name_parts, cluster)) + return false; - bool allowed_on_cluster = allow_on_cluster && name_parts.empty(); - if (allowed_on_cluster) - parseOnCluster(pos, expected, cluster); - - std::vector> database_and_table_names; - bool allowed_multiple_db_and_table_names = ((name_parts.empty()) && (short_names.size() == 1)); - if (allowed_multiple_db_and_table_names) - { - if (!parseOnDatabaseAndTableNames(pos, expected, database_and_table_names)) - return false; - } - else - { - if (!parseOnDatabaseAndTableName(pos, expected, database_and_table_names.emplace_back())) - return false; - } - - allowed_on_cluster &= cluster.empty(); - if (allowed_on_cluster) - parseOnCluster(pos, expected, cluster); - - for (const String & short_name : short_names) - for (const auto & [database, table_name] : database_and_table_names) - name_parts.push_back({short_name, database, table_name}); - - if ((short_names.size() != 1) || (database_and_table_names.size() != 1) || !cluster.empty()) - break; + size_t num_new_name_parts = new_name_parts.size(); + assert(num_new_name_parts >= 1); + boost::range::push_back(name_parts, std::move(new_name_parts)); + if ((num_new_name_parts != 1) || !cluster.empty()) + break; } while (ParserToken{TokenType::Comma}.ignore(pos, expected)); diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index 9f5267b3a9b..8999fa8bb47 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -31,7 +31,7 @@ NamesAndTypesList StorageSystemRowPolicies::getNamesAndTypes() {"database", std::make_shared()}, {"table", std::make_shared()}, {"id", std::make_shared()}, - {"source", std::make_shared()}, + {"storage", std::make_shared()}, }; for (auto type : ext::range(MAX_CONDITION_TYPE)) From a5b70fbdda5957d9ab4ad9c45e872813f1a033af Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 2 Jun 2020 00:43:16 +0300 Subject: [PATCH 237/329] Add tests. --- .../0_stateless/01292_create_user.reference | 108 +++++++++ .../queries/0_stateless/01292_create_user.sql | 211 ++++++++++++++++++ .../0_stateless/01293_create_role.reference | 38 ++++ .../queries/0_stateless/01293_create_role.sql | 74 ++++++ .../01294_create_settings_profile.reference | 56 +++++ .../01294_create_settings_profile.sql | 103 +++++++++ .../01295_create_row_policy.reference | 35 +++ .../0_stateless/01295_create_row_policy.sql | 79 +++++++ ...e_row_policy_in_current_database.reference | 20 ++ ..._create_row_policy_in_current_database.sql | 51 +++++ .../0_stateless/01297_create_quota.reference | 63 ++++++ .../0_stateless/01297_create_quota.sql | 129 +++++++++++ 12 files changed, 967 insertions(+) create mode 100644 tests/queries/0_stateless/01292_create_user.reference create mode 100644 tests/queries/0_stateless/01292_create_user.sql create mode 100644 tests/queries/0_stateless/01293_create_role.reference create mode 100644 tests/queries/0_stateless/01293_create_role.sql create mode 100644 tests/queries/0_stateless/01294_create_settings_profile.reference create mode 100644 tests/queries/0_stateless/01294_create_settings_profile.sql create mode 100644 tests/queries/0_stateless/01295_create_row_policy.reference create mode 100644 tests/queries/0_stateless/01295_create_row_policy.sql create mode 100644 tests/queries/0_stateless/01296_create_row_policy_in_current_database.reference create mode 100644 tests/queries/0_stateless/01296_create_row_policy_in_current_database.sql create mode 100644 tests/queries/0_stateless/01297_create_quota.reference create mode 100644 tests/queries/0_stateless/01297_create_quota.sql diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference new file mode 100644 index 00000000000..555bd99bd94 --- /dev/null +++ b/tests/queries/0_stateless/01292_create_user.reference @@ -0,0 +1,108 @@ +-- default +CREATE USER u1_01292 +-- same as default +CREATE USER u2_01292 +CREATE USER u3_01292 +-- rename +CREATE USER u2_01292_renamed +-- authentication +CREATE USER u1_01292 +CREATE USER u2_01292 IDENTIFIED WITH plaintext_password +CREATE USER u3_01292 IDENTIFIED WITH sha256_password +CREATE USER u4_01292 IDENTIFIED WITH sha256_password +CREATE USER u5_01292 IDENTIFIED WITH sha256_password +CREATE USER u6_01292 IDENTIFIED WITH double_sha1_password +CREATE USER u7_01292 IDENTIFIED WITH double_sha1_password +CREATE USER u1_01292 IDENTIFIED WITH sha256_password +CREATE USER u2_01292 IDENTIFIED WITH sha256_password +CREATE USER u3_01292 IDENTIFIED WITH sha256_password +CREATE USER u4_01292 IDENTIFIED WITH plaintext_password +CREATE USER u5_01292 +-- host +CREATE USER u1_01292 +CREATE USER u2_01292 HOST NONE +CREATE USER u3_01292 HOST LOCAL +CREATE USER u4_01292 HOST NAME \'myhost.com\' +CREATE USER u5_01292 HOST LOCAL, NAME \'myhost.com\' +CREATE USER u6_01292 HOST LOCAL, NAME \'myhost.com\' +CREATE USER u7_01292 HOST REGEXP \'.*\\\\.myhost\\\\.com\' +CREATE USER u8_01292 +CREATE USER u9_01292 HOST LIKE \'%.myhost.com\' +CREATE USER u10_01292 HOST LIKE \'%.myhost.com\' +CREATE USER u11_01292 HOST LOCAL +CREATE USER u12_01292 HOST IP \'192.168.1.1\' +CREATE USER u13_01292 HOST IP \'192.168.0.0/16\' +CREATE USER u14_01292 HOST LOCAL +CREATE USER u15_01292 HOST IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' +CREATE USER u16_01292 HOST LOCAL, IP \'65:ff0c::/96\' +CREATE USER u1_01292 HOST NONE +CREATE USER u2_01292 HOST NAME \'myhost.com\' +CREATE USER u3_01292 HOST LOCAL, NAME \'myhost.com\' +CREATE USER u4_01292 HOST NONE +-- host after @ +CREATE USER u1_01292 +CREATE USER u1_01292 +CREATE USER `u2_01292@%.myhost.com` HOST LIKE \'%.myhost.com\' +CREATE USER `u2_01292@%.myhost.com` HOST LIKE \'%.myhost.com\' +CREATE USER `u3_01292@192.168.%.%` HOST LIKE \'192.168.%.%\' +CREATE USER `u3_01292@192.168.%.%` HOST LIKE \'192.168.%.%\' +CREATE USER `u4_01292@::1` HOST LOCAL +CREATE USER `u4_01292@::1` HOST LOCAL +CREATE USER `u5_01292@65:ff0c::/96` HOST LIKE \'65:ff0c::/96\' +CREATE USER `u5_01292@65:ff0c::/96` HOST LIKE \'65:ff0c::/96\' +CREATE USER u1_01292 HOST LOCAL +CREATE USER `u2_01292@%.myhost.com` +-- settings +CREATE USER u1_01292 +CREATE USER u2_01292 SETTINGS PROFILE default +CREATE USER u3_01292 SETTINGS max_memory_usage = 5000000 +CREATE USER u4_01292 SETTINGS max_memory_usage MIN 5000000 +CREATE USER u5_01292 SETTINGS max_memory_usage MAX 5000000 +CREATE USER u6_01292 SETTINGS max_memory_usage READONLY +CREATE USER u7_01292 SETTINGS max_memory_usage WRITABLE +CREATE USER u8_01292 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 READONLY +CREATE USER u9_01292 SETTINGS PROFILE default, max_memory_usage = 5000000 WRITABLE +CREATE USER u1_01292 SETTINGS readonly = 1 +CREATE USER u2_01292 SETTINGS readonly = 1 +CREATE USER u3_01292 +-- default role +CREATE USER u1_01292 +CREATE USER u2_01292 DEFAULT ROLE NONE +CREATE USER u3_01292 DEFAULT ROLE r1_01292 +CREATE USER u4_01292 DEFAULT ROLE r1_01292, r2_01292 +CREATE USER u5_01292 DEFAULT ROLE ALL EXCEPT r2_01292 +CREATE USER u6_01292 DEFAULT ROLE ALL EXCEPT r1_01292, r2_01292 +CREATE USER u1_01292 DEFAULT ROLE r1_01292 +CREATE USER u2_01292 DEFAULT ROLE ALL EXCEPT r2_01292 +CREATE USER u3_01292 DEFAULT ROLE r2_01292 +CREATE USER u4_01292 +CREATE USER u5_01292 DEFAULT ROLE ALL EXCEPT r1_01292 +CREATE USER u6_01292 DEFAULT ROLE NONE +-- complex +CREATE USER u1_01292 IDENTIFIED WITH plaintext_password HOST LOCAL SETTINGS readonly = 1 +CREATE USER u1_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE NONE SETTINGS PROFILE default +-- multiple users in one command +CREATE USER u1_01292 DEFAULT ROLE NONE +CREATE USER u2_01292 DEFAULT ROLE NONE +CREATE USER u3_01292 HOST LIKE \'%.%.myhost.com\' +CREATE USER u4_01292 HOST LIKE \'%.%.myhost.com\' +CREATE USER `u5_01292@%.host.com` HOST LIKE \'%.host.com\' +CREATE USER `u6_01292@%.host.com` HOST LIKE \'%.host.com\' +CREATE USER `u7_01292@%.host.com` HOST LIKE \'%.host.com\' +CREATE USER `u8_01292@%.otherhost.com` HOST LIKE \'%.otherhost.com\' +CREATE USER u1_01292 DEFAULT ROLE NONE SETTINGS readonly = 1 +CREATE USER u2_01292 DEFAULT ROLE r1_01292, r2_01292 SETTINGS readonly = 1 +CREATE USER u3_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 +CREATE USER u4_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 +-- system.users +u1_01292 disk plaintext_password [] [] ['localhost'] [] [] 1 [] [] +u2_01292 disk no_password [] [] [] [] ['%.%.myhost.com'] 0 [] [] +u3_01292 disk sha256_password [] ['192.169.1.1','192.168.0.0/16'] ['localhost'] [] [] 0 ['r1_01292'] [] +u4_01292 disk double_sha1_password [] ['::/0'] [] [] [] 1 [] ['r1_01292'] +-- system.settings_profile_elements +\N u1_01292 \N 0 readonly 1 \N \N \N \N +\N u2_01292 \N 0 \N \N \N \N \N default +\N u3_01292 \N 0 max_memory_usage 5000000 4000000 6000000 0 \N +\N u4_01292 \N 0 \N \N \N \N \N default +\N u4_01292 \N 1 max_memory_usage 5000000 \N \N \N \N +\N u4_01292 \N 2 readonly 1 \N \N \N \N diff --git a/tests/queries/0_stateless/01292_create_user.sql b/tests/queries/0_stateless/01292_create_user.sql new file mode 100644 index 00000000000..5ae7f3921e6 --- /dev/null +++ b/tests/queries/0_stateless/01292_create_user.sql @@ -0,0 +1,211 @@ +DROP USER IF EXISTS u1_01292, u2_01292, u3_01292, u4_01292, u5_01292, u6_01292, u7_01292, u8_01292, u9_01292; +DROP USER IF EXISTS u10_01292, u11_01292, u12_01292, u13_01292, u14_01292, u15_01292, u16_01292; +DROP USER IF EXISTS u2_01292_renamed; +DROP USER IF EXISTS u1_01292@'%', 'u2_01292@%.myhost.com', u3_01292@'192.168.%.%', 'u4_01292@::1', u5_01292@'65:ff0c::/96'; +DROP USER IF EXISTS u5_01292@'%.host.com', u6_01292@'%.host.com', u7_01292@'%.host.com', u8_01292@'%.otherhost.com'; +DROP ROLE IF EXISTS r1_01292, r2_01292; + +SELECT '-- default'; +CREATE USER u1_01292; +SHOW CREATE USER u1_01292; + +SELECT '-- same as default'; +CREATE USER u2_01292 NOT IDENTIFIED HOST ANY SETTINGS NONE DEFAULT ROLE ALL; +CREATE USER u3_01292 DEFAULT ROLE ALL IDENTIFIED WITH no_password SETTINGS NONE HOST ANY; +SHOW CREATE USER u2_01292; +SHOW CREATE USER u3_01292; + +SELECT '-- rename'; +ALTER USER u2_01292 RENAME TO 'u2_01292_renamed'; +SHOW CREATE USER u2_01292; -- { serverError 192 } -- User not found +SHOW CREATE USER u2_01292_renamed; +DROP USER u1_01292, u2_01292_renamed, u3_01292; + +SELECT '-- authentication'; +CREATE USER u1_01292 NOT IDENTIFIED; +CREATE USER u2_01292 IDENTIFIED WITH plaintext_password BY 'qwe123'; +CREATE USER u3_01292 IDENTIFIED BY 'qwe123'; +CREATE USER u4_01292 IDENTIFIED WITH sha256_password BY 'qwe123'; +CREATE USER u5_01292 IDENTIFIED WITH sha256_hash BY '18138372FAD4B94533CD4881F03DC6C69296DD897234E0CEE83F727E2E6B1F63'; +CREATE USER u6_01292 IDENTIFIED WITH double_sha1_password BY 'qwe123'; +CREATE USER u7_01292 IDENTIFIED WITH double_sha1_hash BY '8DCDD69CE7D121DE8013062AEAEB2A148910D50E'; +SHOW CREATE USER u1_01292; +SHOW CREATE USER u2_01292; +SHOW CREATE USER u3_01292; +SHOW CREATE USER u4_01292; +SHOW CREATE USER u5_01292; +SHOW CREATE USER u6_01292; +SHOW CREATE USER u7_01292; +ALTER USER u1_01292 IDENTIFIED BY '123qwe'; +ALTER USER u2_01292 IDENTIFIED BY '123qwe'; +ALTER USER u3_01292 IDENTIFIED BY '123qwe'; +ALTER USER u4_01292 IDENTIFIED WITH plaintext_password BY '123qwe'; +ALTER USER u5_01292 NOT IDENTIFIED; +SHOW CREATE USER u1_01292; +SHOW CREATE USER u2_01292; +SHOW CREATE USER u3_01292; +SHOW CREATE USER u4_01292; +SHOW CREATE USER u5_01292; +DROP USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292, u6_01292, u7_01292; + +SELECT '-- host'; +CREATE USER u1_01292 HOST ANY; +CREATE USER u2_01292 HOST NONE; +CREATE USER u3_01292 HOST LOCAL; +CREATE USER u4_01292 HOST NAME 'myhost.com'; +CREATE USER u5_01292 HOST NAME 'myhost.com', LOCAL; +CREATE USER u6_01292 HOST LOCAL, NAME 'myhost.com'; +CREATE USER u7_01292 HOST REGEXP '.*\\.myhost\\.com'; +CREATE USER u8_01292 HOST LIKE '%'; +CREATE USER u9_01292 HOST LIKE '%.myhost.com'; +CREATE USER u10_01292 HOST LIKE '%.myhost.com', '%.myhost2.com'; +CREATE USER u11_01292 HOST IP '127.0.0.1'; +CREATE USER u12_01292 HOST IP '192.168.1.1'; +CREATE USER u13_01292 HOST IP '192.168.0.0/16'; +CREATE USER u14_01292 HOST IP '::1'; +CREATE USER u15_01292 HOST IP '2001:0db8:11a3:09d7:1f34:8a2e:07a0:765d'; +CREATE USER u16_01292 HOST IP '65:ff0c::/96', '::1'; +SHOW CREATE USER u1_01292; +SHOW CREATE USER u2_01292; +SHOW CREATE USER u3_01292; +SHOW CREATE USER u4_01292; +SHOW CREATE USER u5_01292; +SHOW CREATE USER u6_01292; +SHOW CREATE USER u7_01292; +SHOW CREATE USER u8_01292; +SHOW CREATE USER u9_01292; +SHOW CREATE USER u10_01292; +SHOW CREATE USER u11_01292; +SHOW CREATE USER u12_01292; +SHOW CREATE USER u13_01292; +SHOW CREATE USER u14_01292; +SHOW CREATE USER u15_01292; +SHOW CREATE USER u16_01292; +ALTER USER u1_01292 HOST NONE; +ALTER USER u2_01292 HOST NAME 'myhost.com'; +ALTER USER u3_01292 ADD HOST NAME 'myhost.com'; +ALTER USER u4_01292 DROP HOST NAME 'myhost.com'; +SHOW CREATE USER u1_01292; +SHOW CREATE USER u2_01292; +SHOW CREATE USER u3_01292; +SHOW CREATE USER u4_01292; +DROP USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292, u6_01292, u7_01292, u8_01292, u9_01292; +DROP USER u10_01292, u11_01292, u12_01292, u13_01292, u14_01292, u15_01292, u16_01292; + +SELECT '-- host after @'; +CREATE USER u1_01292@'%'; +CREATE USER u2_01292@'%.myhost.com'; +CREATE USER u3_01292@'192.168.%.%'; +CREATE USER u4_01292@'::1'; +CREATE USER u5_01292@'65:ff0c::/96'; +SHOW CREATE USER u1_01292@'%'; +SHOW CREATE USER u1_01292; +SHOW CREATE USER u2_01292@'%.myhost.com'; +SHOW CREATE USER 'u2_01292@%.myhost.com'; +SHOW CREATE USER u3_01292@'192.168.%.%'; +SHOW CREATE USER 'u3_01292@192.168.%.%'; +SHOW CREATE USER u4_01292@'::1'; +SHOW CREATE USER 'u4_01292@::1'; +SHOW CREATE USER u5_01292@'65:ff0c::/96'; +SHOW CREATE USER 'u5_01292@65:ff0c::/96'; +ALTER USER u1_01292@'%' HOST LOCAL; +ALTER USER u2_01292@'%.myhost.com' HOST ANY; +SHOW CREATE USER u1_01292@'%'; +SHOW CREATE USER u2_01292@'%.myhost.com'; +DROP USER u1_01292@'%', 'u2_01292@%.myhost.com', u3_01292@'192.168.%.%', 'u4_01292@::1', u5_01292@'65:ff0c::/96'; + +SELECT '-- settings'; +CREATE USER u1_01292 SETTINGS NONE; +CREATE USER u2_01292 SETTINGS PROFILE 'default'; +CREATE USER u3_01292 SETTINGS max_memory_usage=5000000; +CREATE USER u4_01292 SETTINGS max_memory_usage MIN=5000000; +CREATE USER u5_01292 SETTINGS max_memory_usage MAX=5000000; +CREATE USER u6_01292 SETTINGS max_memory_usage READONLY; +CREATE USER u7_01292 SETTINGS max_memory_usage WRITABLE; +CREATE USER u8_01292 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY; +CREATE USER u9_01292 SETTINGS PROFILE 'default', max_memory_usage=5000000 WRITABLE; +SHOW CREATE USER u1_01292; +SHOW CREATE USER u2_01292; +SHOW CREATE USER u3_01292; +SHOW CREATE USER u4_01292; +SHOW CREATE USER u5_01292; +SHOW CREATE USER u6_01292; +SHOW CREATE USER u7_01292; +SHOW CREATE USER u8_01292; +SHOW CREATE USER u9_01292; +ALTER USER u1_01292 SETTINGS readonly=1; +ALTER USER u2_01292 SETTINGS readonly=1; +ALTER USER u3_01292 SETTINGS NONE; +SHOW CREATE USER u1_01292; +SHOW CREATE USER u2_01292; +SHOW CREATE USER u3_01292; +DROP USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292, u6_01292, u7_01292, u8_01292, u9_01292; + +SELECT '-- default role'; +CREATE ROLE r1_01292, r2_01292; +CREATE USER u1_01292 DEFAULT ROLE ALL; +CREATE USER u2_01292 DEFAULT ROLE NONE; +CREATE USER u3_01292 DEFAULT ROLE r1_01292; +CREATE USER u4_01292 DEFAULT ROLE r1_01292, r2_01292; +CREATE USER u5_01292 DEFAULT ROLE ALL EXCEPT r2_01292; +CREATE USER u6_01292 DEFAULT ROLE ALL EXCEPT r1_01292, r2_01292; +SHOW CREATE USER u1_01292; +SHOW CREATE USER u2_01292; +SHOW CREATE USER u3_01292; +SHOW CREATE USER u4_01292; +SHOW CREATE USER u5_01292; +SHOW CREATE USER u6_01292; +GRANT r1_01292, r2_01292 TO u1_01292, u2_01292, u3_01292, u4_01292, u5_01292, u6_01292; +ALTER USER u1_01292 DEFAULT ROLE r1_01292; +ALTER USER u2_01292 DEFAULT ROLE ALL EXCEPT r2_01292; +SET DEFAULT ROLE r2_01292 TO u3_01292; +SET DEFAULT ROLE ALL TO u4_01292; +SET DEFAULT ROLE ALL EXCEPT r1_01292 TO u5_01292; +SET DEFAULT ROLE NONE TO u6_01292; +SHOW CREATE USER u1_01292; +SHOW CREATE USER u2_01292; +SHOW CREATE USER u3_01292; +SHOW CREATE USER u4_01292; +SHOW CREATE USER u5_01292; +SHOW CREATE USER u6_01292; +DROP USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292, u6_01292; + +SELECT '-- complex'; +CREATE USER u1_01292 IDENTIFIED WITH plaintext_password BY 'qwe123' HOST LOCAL SETTINGS readonly=1; +SHOW CREATE USER u1_01292; +ALTER USER u1_01292 NOT IDENTIFIED HOST LIKE '%.%.myhost.com' DEFAULT ROLE NONE SETTINGS PROFILE 'default'; +SHOW CREATE USER u1_01292; +DROP USER u1_01292; + +SELECT '-- multiple users in one command'; +CREATE USER u1_01292, u2_01292 DEFAULT ROLE NONE; +CREATE USER u3_01292, u4_01292 HOST LIKE '%.%.myhost.com'; +CREATE USER u5_01292@'%.host.com', u6_01292@'%.host.com'; +CREATE USER u7_01292@'%.host.com', u8_01292@'%.otherhost.com'; +SHOW CREATE USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292@'%.host.com', u6_01292@'%.host.com'; +SHOW CREATE USER u7_01292@'%.host.com', u8_01292@'%.otherhost.com'; +ALTER USER u1_01292, u2_01292 SETTINGS readonly=1; +GRANT r1_01292, r2_01292 TO u2_01292, u3_01292, u4_01292; +SET DEFAULT ROLE r1_01292, r2_01292 TO u2_01292, u3_01292, u4_01292; +SHOW CREATE USER u1_01292, u2_01292, u3_01292, u4_01292; +DROP USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292@'%.host.com', u6_01292@'%.host.com'; +DROP USER u7_01292@'%.host.com', u8_01292@'%.otherhost.com'; + +SELECT '-- system.users'; +CREATE USER u1_01292 IDENTIFIED WITH plaintext_password BY 'qwe123' HOST LOCAL; +CREATE USER u2_01292 NOT IDENTIFIED HOST LIKE '%.%.myhost.com' DEFAULT ROLE NONE; +CREATE USER u3_01292 IDENTIFIED BY 'qwe123' HOST IP '192.168.0.0/16', '192.169.1.1', '::1' DEFAULT ROLE r1_01292; +CREATE USER u4_01292 IDENTIFIED WITH double_sha1_password BY 'qwe123' HOST ANY DEFAULT ROLE ALL EXCEPT r1_01292; +SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except FROM system.users WHERE name LIKE 'u%\_01292' ORDER BY name; +DROP USER u1_01292, u2_01292, u3_01292, u4_01292; + +SELECT '-- system.settings_profile_elements'; +CREATE USER u1_01292 SETTINGS readonly=1; +CREATE USER u2_01292 SETTINGS PROFILE 'default'; +CREATE USER u3_01292 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 WRITABLE; +CREATE USER u4_01292 SETTINGS PROFILE 'default', max_memory_usage=5000000, readonly=1; +CREATE USER u5_01292 SETTINGS NONE; +SELECT * FROM system.settings_profile_elements WHERE user_name LIKE 'u%\_01292' ORDER BY user_name, index; +DROP USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292; + +DROP ROLE r1_01292, r2_01292; diff --git a/tests/queries/0_stateless/01293_create_role.reference b/tests/queries/0_stateless/01293_create_role.reference new file mode 100644 index 00000000000..0cba719af66 --- /dev/null +++ b/tests/queries/0_stateless/01293_create_role.reference @@ -0,0 +1,38 @@ +-- default +CREATE ROLE r1_01293 +-- same as default +CREATE ROLE r2_01293 +-- rename +CREATE ROLE r2_01293_renamed +-- host after @ +CREATE ROLE r1_01293 +CREATE ROLE r1_01293 +CREATE ROLE `r2_01293@%.myhost.com` +CREATE ROLE `r2_01293@%.myhost.com` +-- settings +CREATE ROLE r1_01293 +CREATE ROLE r2_01293 SETTINGS PROFILE default +CREATE ROLE r3_01293 SETTINGS max_memory_usage = 5000000 +CREATE ROLE r4_01293 SETTINGS max_memory_usage MIN 5000000 +CREATE ROLE r5_01293 SETTINGS max_memory_usage MAX 5000000 +CREATE ROLE r6_01293 SETTINGS max_memory_usage READONLY +CREATE ROLE r7_01293 SETTINGS max_memory_usage WRITABLE +CREATE ROLE r8_01293 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 READONLY +CREATE ROLE r9_01293 SETTINGS PROFILE default, max_memory_usage = 5000000 WRITABLE +CREATE ROLE r1_01293 SETTINGS readonly = 1 +CREATE ROLE r2_01293 SETTINGS readonly = 1 +CREATE ROLE r3_01293 +-- multiple roles in one command +CREATE ROLE r1_01293 +CREATE ROLE r2_01293 +CREATE ROLE r1_01293 SETTINGS readonly = 1 +CREATE ROLE r2_01293 SETTINGS readonly = 1 +-- system.roles +r1_01293 disk +-- system.settings_profile_elements +\N \N r1_01293 0 readonly 1 \N \N \N \N +\N \N r2_01293 0 \N \N \N \N \N default +\N \N r3_01293 0 max_memory_usage 5000000 4000000 6000000 0 \N +\N \N r4_01293 0 \N \N \N \N \N default +\N \N r4_01293 1 max_memory_usage 5000000 \N \N \N \N +\N \N r4_01293 2 readonly 1 \N \N \N \N diff --git a/tests/queries/0_stateless/01293_create_role.sql b/tests/queries/0_stateless/01293_create_role.sql new file mode 100644 index 00000000000..963a1020e3f --- /dev/null +++ b/tests/queries/0_stateless/01293_create_role.sql @@ -0,0 +1,74 @@ +DROP ROLE IF EXISTS r1_01293, r2_01293, r3_01293, r4_01293, r5_01293, r6_01293, r7_01293, r8_01293, r9_01293; +DROP ROLE IF EXISTS r2_01293_renamed; +DROP ROLE IF EXISTS r1_01293@'%', 'r2_01293@%.myhost.com'; + +SELECT '-- default'; +CREATE ROLE r1_01293; +SHOW CREATE ROLE r1_01293; + +SELECT '-- same as default'; +CREATE ROLE r2_01293 SETTINGS NONE; +SHOW CREATE ROLE r2_01293; + +SELECT '-- rename'; +ALTER ROLE r2_01293 RENAME TO 'r2_01293_renamed'; +SHOW CREATE ROLE r2_01293; -- { serverError 511 } -- Role not found +SHOW CREATE ROLE r2_01293_renamed; +DROP ROLE r1_01293, r2_01293_renamed; + +SELECT '-- host after @'; +CREATE ROLE r1_01293@'%'; +CREATE ROLE r2_01293@'%.myhost.com'; +SHOW CREATE ROLE r1_01293@'%'; +SHOW CREATE ROLE r1_01293; +SHOW CREATE ROLE r2_01293@'%.myhost.com'; +SHOW CREATE ROLE 'r2_01293@%.myhost.com'; +DROP ROLE r1_01293@'%', 'r2_01293@%.myhost.com'; + +SELECT '-- settings'; +CREATE ROLE r1_01293 SETTINGS NONE; +CREATE ROLE r2_01293 SETTINGS PROFILE 'default'; +CREATE ROLE r3_01293 SETTINGS max_memory_usage=5000000; +CREATE ROLE r4_01293 SETTINGS max_memory_usage MIN=5000000; +CREATE ROLE r5_01293 SETTINGS max_memory_usage MAX=5000000; +CREATE ROLE r6_01293 SETTINGS max_memory_usage READONLY; +CREATE ROLE r7_01293 SETTINGS max_memory_usage WRITABLE; +CREATE ROLE r8_01293 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY; +CREATE ROLE r9_01293 SETTINGS PROFILE 'default', max_memory_usage=5000000 WRITABLE; +SHOW CREATE ROLE r1_01293; +SHOW CREATE ROLE r2_01293; +SHOW CREATE ROLE r3_01293; +SHOW CREATE ROLE r4_01293; +SHOW CREATE ROLE r5_01293; +SHOW CREATE ROLE r6_01293; +SHOW CREATE ROLE r7_01293; +SHOW CREATE ROLE r8_01293; +SHOW CREATE ROLE r9_01293; +ALTER ROLE r1_01293 SETTINGS readonly=1; +ALTER ROLE r2_01293 SETTINGS readonly=1; +ALTER ROLE r3_01293 SETTINGS NONE; +SHOW CREATE ROLE r1_01293; +SHOW CREATE ROLE r2_01293; +SHOW CREATE ROLE r3_01293; +DROP ROLE r1_01293, r2_01293, r3_01293, r4_01293, r5_01293, r6_01293, r7_01293, r8_01293, r9_01293; + +SELECT '-- multiple roles in one command'; +CREATE ROLE r1_01293, r2_01293; +SHOW CREATE ROLE r1_01293, r2_01293; +ALTER ROLE r1_01293, r2_01293 SETTINGS readonly=1; +SHOW CREATE ROLE r1_01293, r2_01293; +DROP ROLE r1_01293, r2_01293; + +SELECT '-- system.roles'; +CREATE ROLE r1_01293; +SELECT name, storage from system.roles WHERE name='r1_01293'; +DROP ROLE r1_01293; + +SELECT '-- system.settings_profile_elements'; +CREATE ROLE r1_01293 SETTINGS readonly=1; +CREATE ROLE r2_01293 SETTINGS PROFILE 'default'; +CREATE ROLE r3_01293 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 WRITABLE; +CREATE ROLE r4_01293 SETTINGS PROFILE 'default', max_memory_usage=5000000, readonly=1; +CREATE ROLE r5_01293 SETTINGS NONE; +SELECT * FROM system.settings_profile_elements WHERE role_name LIKE 'r%\_01293' ORDER BY role_name, index; +DROP ROLE r1_01293, r2_01293, r3_01293, r4_01293, r5_01293; diff --git a/tests/queries/0_stateless/01294_create_settings_profile.reference b/tests/queries/0_stateless/01294_create_settings_profile.reference new file mode 100644 index 00000000000..527ceea3dd7 --- /dev/null +++ b/tests/queries/0_stateless/01294_create_settings_profile.reference @@ -0,0 +1,56 @@ +-- default +CREATE SETTINGS PROFILE s1_01294 +-- same as default +CREATE SETTINGS PROFILE s2_01294 +CREATE SETTINGS PROFILE s3_01294 +-- rename +CREATE SETTINGS PROFILE s2_01294_renamed +-- settings +CREATE SETTINGS PROFILE s1_01294 +CREATE SETTINGS PROFILE s2_01294 SETTINGS INHERIT default +CREATE SETTINGS PROFILE s3_01294 SETTINGS max_memory_usage = 5000000 +CREATE SETTINGS PROFILE s4_01294 SETTINGS max_memory_usage MIN 5000000 +CREATE SETTINGS PROFILE s5_01294 SETTINGS max_memory_usage MAX 5000000 +CREATE SETTINGS PROFILE s6_01294 SETTINGS max_memory_usage READONLY +CREATE SETTINGS PROFILE s7_01294 SETTINGS max_memory_usage WRITABLE +CREATE SETTINGS PROFILE s8_01294 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 READONLY +CREATE SETTINGS PROFILE s9_01294 SETTINGS INHERIT default, max_memory_usage = 5000000 WRITABLE +CREATE SETTINGS PROFILE s10_01294 SETTINGS INHERIT s1_01294, INHERIT s3_01294, INHERIT default, readonly = 0, max_memory_usage MAX 6000000 +CREATE SETTINGS PROFILE s1_01294 SETTINGS readonly = 0 +CREATE SETTINGS PROFILE s2_01294 SETTINGS readonly = 1 +CREATE SETTINGS PROFILE s3_01294 +-- to roles +CREATE SETTINGS PROFILE s1_01294 +CREATE SETTINGS PROFILE s2_01294 TO ALL +CREATE SETTINGS PROFILE s3_01294 TO r1_01294 +CREATE SETTINGS PROFILE s4_01294 TO u1_01294 +CREATE SETTINGS PROFILE s5_01294 TO r1_01294, u1_01294 +CREATE SETTINGS PROFILE s6_01294 TO ALL EXCEPT r1_01294 +CREATE SETTINGS PROFILE s7_01294 TO ALL EXCEPT r1_01294, u1_01294 +CREATE SETTINGS PROFILE s1_01294 TO u1_01294 +CREATE SETTINGS PROFILE s2_01294 +-- complex +CREATE SETTINGS PROFILE s1_01294 SETTINGS readonly = 0 TO r1_01294 +CREATE SETTINGS PROFILE s1_01294 SETTINGS INHERIT default +-- multiple profiles in one command +CREATE SETTINGS PROFILE s1_01294 SETTINGS max_memory_usage = 5000000 +CREATE SETTINGS PROFILE s2_01294 SETTINGS max_memory_usage = 5000000 +CREATE SETTINGS PROFILE s3_01294 TO ALL +CREATE SETTINGS PROFILE s4_01294 TO ALL +CREATE SETTINGS PROFILE s1_01294 SETTINGS max_memory_usage = 6000000 +CREATE SETTINGS PROFILE s2_01294 SETTINGS max_memory_usage = 6000000 TO r1_01294 +CREATE SETTINGS PROFILE s3_01294 SETTINGS max_memory_usage = 6000000 TO r1_01294 +CREATE SETTINGS PROFILE s4_01294 TO r1_01294 +-- system.settings_profiles +s1_01294 disk 0 0 [] [] +s2_01294 disk 1 0 ['r1_01294'] [] +s3_01294 disk 1 0 ['r1_01294'] [] +s4_01294 disk 1 0 ['r1_01294'] [] +s5_01294 disk 3 1 [] ['r1_01294'] +-- system.settings_profile_elements +s2_01294 \N \N 0 readonly 0 \N \N \N \N +s3_01294 \N \N 0 max_memory_usage 5000000 4000000 6000000 1 \N +s4_01294 \N \N 0 max_memory_usage 5000000 \N \N \N \N +s5_01294 \N \N 0 \N \N \N \N \N default +s5_01294 \N \N 1 readonly 0 \N \N \N \N +s5_01294 \N \N 2 max_memory_usage \N \N 6000000 0 \N diff --git a/tests/queries/0_stateless/01294_create_settings_profile.sql b/tests/queries/0_stateless/01294_create_settings_profile.sql new file mode 100644 index 00000000000..2d34042f2b4 --- /dev/null +++ b/tests/queries/0_stateless/01294_create_settings_profile.sql @@ -0,0 +1,103 @@ +DROP SETTINGS PROFILE IF EXISTS s1_01294, s2_01294, s3_01294, s4_01294, s5_01294, s6_01294, s7_01294, s8_01294, s9_01294, s10_01294; +DROP SETTINGS PROFILE IF EXISTS s2_01294_renamed; +DROP USER IF EXISTS u1_01294; +DROP ROLE IF EXISTS r1_01294; + +SELECT '-- default'; +CREATE SETTINGS PROFILE s1_01294; +SHOW CREATE SETTINGS PROFILE s1_01294; + +SELECT '-- same as default'; +CREATE SETTINGS PROFILE s2_01294 SETTINGS NONE TO NONE; +CREATE PROFILE s3_01294; +SHOW CREATE PROFILE s2_01294; +SHOW CREATE SETTINGS PROFILE s3_01294; + +SELECT '-- rename'; +ALTER SETTINGS PROFILE s2_01294 RENAME TO 's2_01294_renamed'; +SHOW CREATE SETTINGS PROFILE s2_01294; -- { serverError 180 } -- Profile not found +SHOW CREATE SETTINGS PROFILE s2_01294_renamed; +DROP SETTINGS PROFILE s1_01294, s2_01294_renamed, s3_01294; + +SELECT '-- settings'; +CREATE PROFILE s1_01294 SETTINGS NONE; +CREATE PROFILE s2_01294 SETTINGS INHERIT 'default'; +CREATE PROFILE s3_01294 SETTINGS max_memory_usage=5000000; +CREATE PROFILE s4_01294 SETTINGS max_memory_usage MIN=5000000; +CREATE PROFILE s5_01294 SETTINGS max_memory_usage MAX=5000000; +CREATE PROFILE s6_01294 SETTINGS max_memory_usage READONLY; +CREATE PROFILE s7_01294 SETTINGS max_memory_usage WRITABLE; +CREATE PROFILE s8_01294 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY; +CREATE PROFILE s9_01294 SETTINGS INHERIT 'default', max_memory_usage=5000000 WRITABLE; +CREATE PROFILE s10_01294 SETTINGS INHERIT s1_01294, s3_01294, INHERIT default, readonly=0, max_memory_usage MAX 6000000; +SHOW CREATE PROFILE s1_01294; +SHOW CREATE PROFILE s2_01294; +SHOW CREATE PROFILE s3_01294; +SHOW CREATE PROFILE s4_01294; +SHOW CREATE PROFILE s5_01294; +SHOW CREATE PROFILE s6_01294; +SHOW CREATE PROFILE s7_01294; +SHOW CREATE PROFILE s8_01294; +SHOW CREATE PROFILE s9_01294; +SHOW CREATE PROFILE s10_01294; +ALTER PROFILE s1_01294 SETTINGS readonly=0; +ALTER PROFILE s2_01294 SETTINGS readonly=1; +ALTER PROFILE s3_01294 SETTINGS NONE; +SHOW CREATE PROFILE s1_01294; +SHOW CREATE PROFILE s2_01294; +SHOW CREATE PROFILE s3_01294; +DROP PROFILE s1_01294, s2_01294, s3_01294, s4_01294, s5_01294, s6_01294, s7_01294, s8_01294, s9_01294, s10_01294; + +SELECT '-- to roles'; +CREATE ROLE r1_01294; +CREATE USER u1_01294; +CREATE PROFILE s1_01294 TO NONE; +CREATE PROFILE s2_01294 TO ALL; +CREATE PROFILE s3_01294 TO r1_01294; +CREATE PROFILE s4_01294 TO u1_01294; +CREATE PROFILE s5_01294 TO r1_01294, u1_01294; +CREATE PROFILE s6_01294 TO ALL EXCEPT r1_01294; +CREATE PROFILE s7_01294 TO ALL EXCEPT r1_01294, u1_01294; +SHOW CREATE PROFILE s1_01294; +SHOW CREATE PROFILE s2_01294; +SHOW CREATE PROFILE s3_01294; +SHOW CREATE PROFILE s4_01294; +SHOW CREATE PROFILE s5_01294; +SHOW CREATE PROFILE s6_01294; +SHOW CREATE PROFILE s7_01294; +ALTER PROFILE s1_01294 TO u1_01294; +ALTER PROFILE s2_01294 TO NONE; +SHOW CREATE PROFILE s1_01294; +SHOW CREATE PROFILE s2_01294; +DROP PROFILE s1_01294, s2_01294, s3_01294, s4_01294, s5_01294, s6_01294, s7_01294; + +SELECT '-- complex'; +CREATE SETTINGS PROFILE s1_01294 SETTINGS readonly=0 TO r1_01294; +SHOW CREATE SETTINGS PROFILE s1_01294; +ALTER SETTINGS PROFILE s1_01294 SETTINGS INHERIT 'default' TO NONE; +SHOW CREATE SETTINGS PROFILE s1_01294; +DROP SETTINGS PROFILE s1_01294; + +SELECT '-- multiple profiles in one command'; +CREATE PROFILE s1_01294, s2_01294 SETTINGS max_memory_usage=5000000; +CREATE PROFILE s3_01294, s4_01294 TO ALL; +SHOW CREATE PROFILE s1_01294, s2_01294, s3_01294, s4_01294; +ALTER PROFILE s1_01294, s2_01294, s3_01294 SETTINGS max_memory_usage=6000000; +ALTER PROFILE s2_01294, s3_01294, s4_01294 TO r1_01294; +SHOW CREATE PROFILE s1_01294, s2_01294, s3_01294, s4_01294; +DROP PROFILE s1_01294, s2_01294, s3_01294, s4_01294; + +SELECT '-- system.settings_profiles'; +CREATE PROFILE s1_01294; +CREATE PROFILE s2_01294 SETTINGS readonly=0 TO r1_01294;; +CREATE PROFILE s3_01294 SETTINGS max_memory_usage=5000000 MIN 4000000 MAX 6000000 READONLY TO r1_01294; +CREATE PROFILE s4_01294 SETTINGS max_memory_usage=5000000 TO r1_01294; +CREATE PROFILE s5_01294 SETTINGS INHERIT default, readonly=0, max_memory_usage MAX 6000000 WRITABLE TO ALL EXCEPT r1_01294; +SELECT name, storage, num_elements, apply_to_all, apply_to_list, apply_to_except FROM system.settings_profiles WHERE name LIKE 's%\_01294' ORDER BY name; + +SELECT '-- system.settings_profile_elements'; +SELECT * FROM system.settings_profile_elements WHERE profile_name LIKE 's%\_01294' ORDER BY profile_name, index; +DROP PROFILE s1_01294, s2_01294, s3_01294, s4_01294, s5_01294; + +DROP ROLE r1_01294; +DROP USER u1_01294; diff --git a/tests/queries/0_stateless/01295_create_row_policy.reference b/tests/queries/0_stateless/01295_create_row_policy.reference new file mode 100644 index 00000000000..2ed894c923e --- /dev/null +++ b/tests/queries/0_stateless/01295_create_row_policy.reference @@ -0,0 +1,35 @@ +-- default +CREATE ROW POLICY p1_01295 ON db.table +-- same as default +CREATE ROW POLICY p2_01295 ON db.table +CREATE ROW POLICY p3_01295 ON db.table +-- rename +CREATE ROW POLICY p2_01295_renamed ON db.table +-- filter +CREATE ROW POLICY p1_01295 ON db.table FOR SELECT USING (a < b) AND (c > d) +CREATE ROW POLICY p2_01295 ON db.table AS restrictive FOR SELECT USING id = currentUser() +CREATE ROW POLICY p3_01295 ON db.table FOR SELECT USING 1 +CREATE ROW POLICY p1_01295 ON db.table AS restrictive FOR SELECT USING 0 +-- to roles +CREATE ROW POLICY p1_01295 ON db.table +CREATE ROW POLICY p2_01295 ON db.table TO ALL +CREATE ROW POLICY p3_01295 ON db.table TO r1_01295 +CREATE ROW POLICY p4_01295 ON db.table TO u1_01295 +CREATE ROW POLICY p5_01295 ON db.table TO r1_01295, u1_01295 +CREATE ROW POLICY p6_01295 ON db.table TO ALL EXCEPT r1_01295 +CREATE ROW POLICY p7_01295 ON db.table TO ALL EXCEPT r1_01295, u1_01295 +CREATE ROW POLICY p1_01295 ON db.table TO u1_01295 +CREATE ROW POLICY p2_01295 ON db.table +-- multiple policies in one command +CREATE ROW POLICY p1_01295 ON db.table FOR SELECT USING 1 +CREATE ROW POLICY p2_01295 ON db.table FOR SELECT USING 1 +CREATE ROW POLICY p3_01295 ON db.table TO u1_01295 +CREATE ROW POLICY p3_01295 ON db2.table2 TO u1_01295 +CREATE ROW POLICY p4_01295 ON db.table FOR SELECT USING a = b +CREATE ROW POLICY p5_01295 ON db2.table2 FOR SELECT USING a = b +CREATE ROW POLICY p1_01295 ON db.table FOR SELECT USING 1 TO ALL +CREATE ROW POLICY p2_01295 ON db.table FOR SELECT USING 1 TO ALL +-- system.row_policies +p1_01295 ON db.table p1_01295 db table disk (a < b) AND (c > d) 0 0 [] [] +p2_01295 ON db.table p2_01295 db table disk id = currentUser() 1 0 ['u1_01295'] [] +p3_01295 ON db.table p3_01295 db table disk 1 0 1 [] ['r1_01295'] diff --git a/tests/queries/0_stateless/01295_create_row_policy.sql b/tests/queries/0_stateless/01295_create_row_policy.sql new file mode 100644 index 00000000000..b484d0ea0f3 --- /dev/null +++ b/tests/queries/0_stateless/01295_create_row_policy.sql @@ -0,0 +1,79 @@ +DROP ROW POLICY IF EXISTS p1_01295, p2_01295, p3_01295, p4_01295, p5_01295, p6_01295, p7_01295, p8_01295, p9_01295, p10_01295 ON db.table; +DROP ROW POLICY IF EXISTS p2_01295_renamed ON db.table; +DROP ROW POLICY IF EXISTS p3_01295 ON db.table, db2.table2; +DROP ROW POLICY IF EXISTS p4_01295 ON db.table, p5_01295 ON db2.table2; +DROP USER IF EXISTS u1_01295; +DROP ROLE IF EXISTS r1_01295; + +SELECT '-- default'; +CREATE ROW POLICY p1_01295 ON db.table; +SHOW CREATE ROW POLICY p1_01295 ON db.table; + +SELECT '-- same as default'; +CREATE ROW POLICY p2_01295 ON db.table USING NONE TO NONE; +CREATE POLICY p3_01295 ON db.table; +SHOW CREATE POLICY p2_01295 ON db.table; +SHOW CREATE ROW POLICY p3_01295 ON db.table; + +SELECT '-- rename'; +ALTER ROW POLICY p2_01295 ON db.table RENAME TO 'p2_01295_renamed'; +SHOW CREATE ROW POLICY p2_01295 ON db.table; -- { serverError 523 } -- Policy not found +SHOW CREATE ROW POLICY p2_01295_renamed ON db.table; +DROP ROW POLICY p1_01295, p2_01295_renamed, p3_01295 ON db.table; + +SELECT '-- filter'; +CREATE ROW POLICY p1_01295 ON db.table USING ad; +CREATE ROW POLICY p2_01295 ON db.table USING id=currentUser() AS RESTRICTIVE; +CREATE ROW POLICY p3_01295 ON db.table USING 1 AS PERMISSIVE; +SHOW CREATE POLICY p1_01295 ON db.table; +SHOW CREATE POLICY p2_01295 ON db.table; +SHOW CREATE POLICY p3_01295 ON db.table; +ALTER ROW POLICY p1_01295 ON db.table FOR SELECT USING 0 AS RESTRICTIVE; +SHOW CREATE POLICY p1_01295 ON db.table; +DROP ROW POLICY p1_01295, p2_01295, p3_01295 ON db.table; + +SELECT '-- to roles'; +CREATE ROLE r1_01295; +CREATE USER u1_01295; +CREATE POLICY p1_01295 ON db.table TO NONE; +CREATE POLICY p2_01295 ON db.table TO ALL; +CREATE POLICY p3_01295 ON db.table TO r1_01295; +CREATE POLICY p4_01295 ON db.table TO u1_01295; +CREATE POLICY p5_01295 ON db.table TO r1_01295, u1_01295; +CREATE POLICY p6_01295 ON db.table TO ALL EXCEPT r1_01295; +CREATE POLICY p7_01295 ON db.table TO ALL EXCEPT r1_01295, u1_01295; +SHOW CREATE POLICY p1_01295 ON db.table; +SHOW CREATE POLICY p2_01295 ON db.table; +SHOW CREATE POLICY p3_01295 ON db.table; +SHOW CREATE POLICY p4_01295 ON db.table; +SHOW CREATE POLICY p5_01295 ON db.table; +SHOW CREATE POLICY p6_01295 ON db.table; +SHOW CREATE POLICY p7_01295 ON db.table; +ALTER POLICY p1_01295 ON db.table TO u1_01295; +ALTER POLICY p2_01295 ON db.table TO NONE; +SHOW CREATE POLICY p1_01295 ON db.table; +SHOW CREATE POLICY p2_01295 ON db.table; +DROP POLICY p1_01295, p2_01295, p3_01295, p4_01295, p5_01295, p6_01295, p7_01295 ON db.table; + +SELECT '-- multiple policies in one command'; +CREATE ROW POLICY p1_01295, p2_01295 ON db.table USING 1; +CREATE ROW POLICY p3_01295 ON db.table, db2.table2 TO u1_01295; +CREATE ROW POLICY p4_01295 ON db.table, p5_01295 ON db2.table2 USING a=b; +SHOW CREATE POLICY p1_01295, p2_01295 ON db.table; +SHOW CREATE POLICY p3_01295 ON db.table, db2.table2; +SHOW CREATE POLICY p4_01295 ON db.table, p5_01295 ON db2.table2; +ALTER POLICY p1_01295, p2_01295 ON db.table TO ALL; +SHOW CREATE POLICY p1_01295, p2_01295 ON db.table; +DROP POLICY p1_01295, p2_01295 ON db.table; +DROP POLICY p3_01295 ON db.table, db2.table2; +DROP POLICY p4_01295 ON db.table, p5_01295 ON db2.table2; + +SELECT '-- system.row_policies'; +CREATE ROW POLICY p1_01295 ON db.table USING ad; +CREATE ROW POLICY p2_01295 ON db.table USING id=currentUser() AS RESTRICTIVE TO u1_01295; +CREATE ROW POLICY p3_01295 ON db.table USING 1 AS PERMISSIVE TO ALL EXCEPT r1_01295; +SELECT name, short_name, database, table, storage, select_filter, is_restrictive, apply_to_all, apply_to_list, apply_to_except from system.row_policies WHERE short_name LIKE 'p%\_01295' ORDER BY name; +DROP ROW POLICY p1_01295, p2_01295, p3_01295 ON db.table; + +DROP ROLE r1_01295; +DROP USER u1_01295; diff --git a/tests/queries/0_stateless/01296_create_row_policy_in_current_database.reference b/tests/queries/0_stateless/01296_create_row_policy_in_current_database.reference new file mode 100644 index 00000000000..fa9c2f73021 --- /dev/null +++ b/tests/queries/0_stateless/01296_create_row_policy_in_current_database.reference @@ -0,0 +1,20 @@ +-- one policy +CREATE ROW POLICY p1_01296 ON db_01296.table +CREATE ROW POLICY p1_01296 ON db_01296.table +CREATE ROW POLICY p1_01296 ON db_01296.table FOR SELECT USING 1 +CREATE ROW POLICY p1_01296 ON db_01296.table FOR SELECT USING 1 +-- multiple policies +CREATE ROW POLICY p1_01296 ON db_01296.table FOR SELECT USING 1 +CREATE ROW POLICY p2_01296 ON db_01296.table FOR SELECT USING 1 +CREATE ROW POLICY p3_01296 ON db_01296.table TO u1_01296 +CREATE ROW POLICY p3_01296 ON db_01296.table2 TO u1_01296 +CREATE ROW POLICY p4_01296 ON db_01296.table FOR SELECT USING a = b +CREATE ROW POLICY p5_01296 ON db_01296.table2 FOR SELECT USING a = b +CREATE ROW POLICY p1_01296 ON db_01296.table FOR SELECT USING 1 +CREATE ROW POLICY p2_01296 ON db_01296.table FOR SELECT USING 1 +CREATE ROW POLICY p3_01296 ON db_01296.table TO u1_01296 +CREATE ROW POLICY p3_01296 ON db_01296.table2 TO u1_01296 +CREATE ROW POLICY p4_01296 ON db_01296.table FOR SELECT USING a = b +CREATE ROW POLICY p5_01296 ON db_01296.table2 FOR SELECT USING a = b +CREATE ROW POLICY p1_01296 ON db_01296.table FOR SELECT USING 1 TO ALL +CREATE ROW POLICY p2_01296 ON db_01296.table FOR SELECT USING 1 TO ALL diff --git a/tests/queries/0_stateless/01296_create_row_policy_in_current_database.sql b/tests/queries/0_stateless/01296_create_row_policy_in_current_database.sql new file mode 100644 index 00000000000..fca570b5651 --- /dev/null +++ b/tests/queries/0_stateless/01296_create_row_policy_in_current_database.sql @@ -0,0 +1,51 @@ +DROP ROW POLICY IF EXISTS p1_01296, p2_01296, p3_01296, p4_01296, p5_01296 ON db_01296.table; +DROP ROW POLICY IF EXISTS p3_01296, p5_01296 ON db_01296.table2; +DROP DATABASE IF EXISTS db_01296; +DROP USER IF EXISTS u1_01296; + +CREATE DATABASE db_01296; +USE db_01296; + +SELECT '-- one policy'; +CREATE POLICY p1_01296 ON table; +SHOW CREATE POLICY p1_01296 ON db_01296.table; +SHOW CREATE POLICY p1_01296 ON table; +ALTER POLICY p1_01296 ON table USING 1; +SHOW CREATE POLICY p1_01296 ON db_01296.table; +SHOW CREATE POLICY p1_01296 ON table; +DROP POLICY p1_01296 ON table; +DROP POLICY p1_01296 ON db_01296.table; -- { serverError 523 } -- Policy not found + +SELECT '-- multiple policies'; +CREATE ROW POLICY p1_01296, p2_01296 ON table USING 1; +CREATE USER u1_01296; +CREATE ROW POLICY p3_01296 ON table, table2 TO u1_01296; +CREATE ROW POLICY p4_01296 ON table, p5_01296 ON table2 USING a=b; +SHOW CREATE POLICY p1_01296 ON table; +SHOW CREATE POLICY p2_01296 ON table; +SHOW CREATE POLICY p3_01296 ON table; +SHOW CREATE POLICY p3_01296 ON table2; +SHOW CREATE POLICY p4_01296 ON table; +SHOW CREATE POLICY p5_01296 ON table2; +SHOW CREATE POLICY p1_01296 ON db_01296.table; +SHOW CREATE POLICY p2_01296 ON db_01296.table; +SHOW CREATE POLICY p3_01296 ON db_01296.table; +SHOW CREATE POLICY p3_01296 ON db_01296.table2; +SHOW CREATE POLICY p4_01296 ON db_01296.table; +SHOW CREATE POLICY p5_01296 ON db_01296.table2; +ALTER POLICY p1_01296, p2_01296 ON table TO ALL; +SHOW CREATE POLICY p1_01296 ON table; +SHOW CREATE POLICY p2_01296 ON table; +DROP POLICY p1_01296, p2_01296 ON table; +DROP POLICY p3_01296 ON table, table2; +DROP POLICY p4_01296 ON table, p5_01296 ON table2; +DROP POLICY p1_01296 ON db_01296.table; -- { serverError 523 } -- Policy not found +DROP POLICY p2_01296 ON db_01296.table; -- { serverError 523 } -- Policy not found +DROP POLICY p3_01296 ON db_01296.table; -- { serverError 523 } -- Policy not found +DROP POLICY p3_01296 ON db_01296.table2; -- { serverError 523 } -- Policy not found +DROP POLICY p4_01296 ON db_01296.table; -- { serverError 523 } -- Policy not found +DROP POLICY p5_01296 ON db_01296.table2; -- { serverError 523 } -- Policy not found + +USE default; +DROP DATABASE db_01296; +DROP USER u1_01296; diff --git a/tests/queries/0_stateless/01297_create_quota.reference b/tests/queries/0_stateless/01297_create_quota.reference new file mode 100644 index 00000000000..b58d3f0f390 --- /dev/null +++ b/tests/queries/0_stateless/01297_create_quota.reference @@ -0,0 +1,63 @@ +-- default +CREATE QUOTA q1_01297 +-- same as default +CREATE QUOTA q2_01297 +CREATE QUOTA q3_01297 +CREATE QUOTA q4_01297 +-- rename +CREATE QUOTA q2_01297_renamed +-- key +CREATE QUOTA q1_01297 +CREATE QUOTA q2_01297 KEYED BY user_name +CREATE QUOTA q3_01297 KEYED BY ip_address +CREATE QUOTA q4_01297 KEYED BY client_key +CREATE QUOTA q5_01297 KEYED BY client_key, user_name +CREATE QUOTA q6_01297 KEYED BY client_key, ip_address +CREATE QUOTA q7_01297 +CREATE QUOTA q8_01297 KEYED BY user_name +CREATE QUOTA q9_01297 KEYED BY ip_address +CREATE QUOTA q10_01297 KEYED BY client_key +CREATE QUOTA q11_01297 KEYED BY client_key, user_name +CREATE QUOTA q12_01297 KEYED BY client_key, ip_address +CREATE QUOTA q1_01297 KEYED BY user_name +CREATE QUOTA q2_01297 KEYED BY client_key, user_name +CREATE QUOTA q3_01297 +-- intervals +CREATE QUOTA q1_01297 FOR INTERVAL 5 day MAX errors = 3 +CREATE QUOTA q2_01297 FOR INTERVAL 30 minute MAX errors = 4 +CREATE QUOTA q3_01297 FOR INTERVAL 1 hour MAX errors = 5 +CREATE QUOTA q4_01297 FOR INTERVAL 2000 second MAX errors = 5 +CREATE QUOTA q5_01297 FOR RANDOMIZED INTERVAL 1 year MAX queries = 100, errors = 11 +CREATE QUOTA q6_01297 FOR INTERVAL 2 month MAX queries = 100, errors = 11, result_rows = 1000, result_bytes = 10000, read_rows = 1001, read_bytes = 10001, execution_time = 2.5 +CREATE QUOTA q7_01297 FOR INTERVAL 1 quarter MAX queries = 100, errors = 11 +CREATE QUOTA q8_01297 FOR INTERVAL 2 month MAX result_rows = 1002, FOR INTERVAL 2 quarter MAX queries = 100, errors = 11 +CREATE QUOTA q1_01297 +CREATE QUOTA q2_01297 FOR INTERVAL 30 minute TRACKING ONLY +CREATE QUOTA q3_01297 FOR INTERVAL 1 hour MAX queries = 70, FOR INTERVAL 2 hour MAX errors = 10 +CREATE QUOTA q4_01297 FOR RANDOMIZED INTERVAL 2000 second MAX errors = 5 +CREATE QUOTA q5_01297 FOR INTERVAL 1 year MAX errors = 111 +-- to roles +CREATE QUOTA q1_01297 +CREATE QUOTA q2_01297 TO ALL +CREATE QUOTA q3_01297 TO r1_01297 +CREATE QUOTA q4_01297 TO u1_01297 +CREATE QUOTA q5_01297 TO r1_01297, u1_01297 +CREATE QUOTA q6_01297 TO ALL EXCEPT r1_01297 +CREATE QUOTA q7_01297 TO ALL EXCEPT r1_01297, u1_01297 +CREATE QUOTA q1_01297 TO u1_01297 +CREATE QUOTA q2_01297 +-- multiple quotas in one command +CREATE QUOTA q1_01297 FOR INTERVAL 1 day MAX errors = 5 +CREATE QUOTA q2_01297 FOR INTERVAL 1 day MAX errors = 5 +CREATE QUOTA q1_01297 FOR INTERVAL 1 day TRACKING ONLY TO r1_01297 +CREATE QUOTA q2_01297 FOR INTERVAL 1 day TRACKING ONLY TO r1_01297 +-- system.quotas +q1_01297 disk ['user_name'] [] 0 ['r1_01297'] [] +q2_01297 disk [] [5259492] 0 ['r1_01297','u1_01297'] [] +q3_01297 disk ['client_key','user_name'] [5259492,15778476] 0 [] [] +q4_01297 disk [] [604800] 1 [] ['u1_01297'] +-- system.quota_limits +q2_01297 5259492 0 100 11 1000 10000 1001 10001 2.5 +q3_01297 5259492 0 \N \N 1002 \N \N \N \N +q3_01297 15778476 0 100 11 \N \N \N \N \N +q4_01297 604800 0 \N \N \N \N \N \N \N diff --git a/tests/queries/0_stateless/01297_create_quota.sql b/tests/queries/0_stateless/01297_create_quota.sql new file mode 100644 index 00000000000..a3fb8331e16 --- /dev/null +++ b/tests/queries/0_stateless/01297_create_quota.sql @@ -0,0 +1,129 @@ +DROP QUOTA IF EXISTS q1_01297, q2_01297, q3_01297, q4_01297, q5_01297, q6_01297, q7_01297, q8_01297, q9_01297, q10_01297; +DROP QUOTA IF EXISTS q11_01297, q12_01297; +DROP QUOTA IF EXISTS q2_01297_renamed; +DROP USER IF EXISTS u1_01297; +DROP ROLE IF EXISTS r1_01297; + +SELECT '-- default'; +CREATE QUOTA q1_01297; +SHOW CREATE QUOTA q1_01297; + +SELECT '-- same as default'; +CREATE QUOTA q2_01297 TO NONE; +CREATE QUOTA q3_01297 FOR INTERVAL 1 HOUR NO LIMITS NOT KEYED TO NONE; +CREATE QUOTA q4_01297 KEYED BY none FOR 1 hour NO LIMITS; +SHOW CREATE QUOTA q2_01297; +SHOW CREATE QUOTA q3_01297; +SHOW CREATE QUOTA q4_01297; + +SELECT '-- rename'; +ALTER QUOTA q2_01297 RENAME TO 'q2_01297_renamed'; +SHOW CREATE QUOTA q2_01297; -- { serverError 199 } -- Policy not found +SHOW CREATE QUOTA q2_01297_renamed; +DROP QUOTA q1_01297, q2_01297_renamed, q3_01297, q4_01297; + +SELECT '-- key'; +CREATE QUOTA q1_01297 NOT KEYED; +CREATE QUOTA q2_01297 KEY BY user_name; +CREATE QUOTA q3_01297 KEY BY ip_address; +CREATE QUOTA q4_01297 KEY BY client_key; +CREATE QUOTA q5_01297 KEY BY client_key, user_name; +CREATE QUOTA q6_01297 KEY BY client_key, ip_address; +CREATE QUOTA q7_01297 KEYED BY 'none'; +CREATE QUOTA q8_01297 KEYED BY 'user name'; +CREATE QUOTA q9_01297 KEYED BY 'IP_ADDRESS'; +CREATE QUOTA q10_01297 KEYED BY CLIENT_KEY; +CREATE QUOTA q11_01297 KEYED BY 'client key or user name'; +CREATE QUOTA q12_01297 KEYED BY 'client key or ip address'; +SHOW CREATE QUOTA q1_01297; +SHOW CREATE QUOTA q2_01297; +SHOW CREATE QUOTA q3_01297; +SHOW CREATE QUOTA q4_01297; +SHOW CREATE QUOTA q5_01297; +SHOW CREATE QUOTA q6_01297; +SHOW CREATE QUOTA q7_01297; +SHOW CREATE QUOTA q8_01297; +SHOW CREATE QUOTA q9_01297; +SHOW CREATE QUOTA q10_01297; +SHOW CREATE QUOTA q11_01297; +SHOW CREATE QUOTA q12_01297; +ALTER QUOTA q1_01297 KEY BY user_name; +ALTER QUOTA q2_01297 KEY BY client_key, user_name; +ALTER QUOTA q3_01297 NOT KEYED; +SHOW CREATE QUOTA q1_01297; +SHOW CREATE QUOTA q2_01297; +SHOW CREATE QUOTA q3_01297; +DROP QUOTA q1_01297, q2_01297, q3_01297, q4_01297, q5_01297, q6_01297, q7_01297, q8_01297, q9_01297, q10_01297, q11_01297, q12_01297; + +SELECT '-- intervals'; +CREATE QUOTA q1_01297 FOR INTERVAL 5 DAY MAX ERRORS = 3; +CREATE QUOTA q2_01297 FOR INTERVAL 30 minute MAX ERRORS 4; +CREATE QUOTA q3_01297 FOR 1 HOUR errors MAX 5; +CREATE QUOTA q4_01297 FOR 2000 SECOND errors MAX 5; +CREATE QUOTA q5_01297 FOR RANDOMIZED INTERVAL 1 YEAR MAX errors = 11, MAX queries = 100; +CREATE QUOTA q6_01297 FOR 2 MONTH MAX errors = 11, queries = 100, result_rows = 1000, result_bytes = 10000, read_rows = 1001, read_bytes = 10001, execution_time=2.5; +CREATE QUOTA q7_01297 FOR 1 QUARTER MAX errors 11, queries 100; +CREATE QUOTA q8_01297 FOR 0.5 year ERRORS MAX 11, QUERIES MAX 100, FOR 2 MONTH RESULT ROWS MAX 1002; +SHOW CREATE QUOTA q1_01297; +SHOW CREATE QUOTA q2_01297; +SHOW CREATE QUOTA q3_01297; +SHOW CREATE QUOTA q4_01297; +SHOW CREATE QUOTA q5_01297; +SHOW CREATE QUOTA q6_01297; +SHOW CREATE QUOTA q7_01297; +SHOW CREATE QUOTA q8_01297; +ALTER QUOTA q1_01297 FOR INTERVAL 5 DAY NO LIMITS; +ALTER QUOTA q2_01297 FOR INTERVAL 30 MINUTE TRACKING ONLY; +ALTER QUOTA q3_01297 FOR INTERVAL 2 HOUR MAX errors = 10, FOR INTERVAL 1 HOUR MAX queries = 70; +ALTER QUOTA q4_01297 FOR RANDOMIZED INTERVAL 2000 SECOND errors MAX 5; +ALTER QUOTA q5_01297 FOR 1 YEAR MAX errors = 111; +SHOW CREATE QUOTA q1_01297; +SHOW CREATE QUOTA q2_01297; +SHOW CREATE QUOTA q3_01297; +SHOW CREATE QUOTA q4_01297; +SHOW CREATE QUOTA q5_01297; +DROP QUOTA q1_01297, q2_01297, q3_01297, q4_01297, q5_01297, q6_01297, q7_01297, q8_01297; + +SELECT '-- to roles'; +CREATE ROLE r1_01297; +CREATE USER u1_01297; +CREATE QUOTA q1_01297 TO NONE; +CREATE QUOTA q2_01297 TO ALL; +CREATE QUOTA q3_01297 TO r1_01297; +CREATE QUOTA q4_01297 TO u1_01297; +CREATE QUOTA q5_01297 TO r1_01297, u1_01297; +CREATE QUOTA q6_01297 TO ALL EXCEPT r1_01297; +CREATE QUOTA q7_01297 TO ALL EXCEPT r1_01297, u1_01297; +SHOW CREATE QUOTA q1_01297; +SHOW CREATE QUOTA q2_01297; +SHOW CREATE QUOTA q3_01297; +SHOW CREATE QUOTA q4_01297; +SHOW CREATE QUOTA q5_01297; +SHOW CREATE QUOTA q6_01297; +SHOW CREATE QUOTA q7_01297; +ALTER QUOTA q1_01297 TO u1_01297; +ALTER QUOTA q2_01297 TO NONE; +SHOW CREATE QUOTA q1_01297; +SHOW CREATE QUOTA q2_01297; +DROP QUOTA q1_01297, q2_01297, q3_01297, q4_01297, q5_01297, q6_01297, q7_01297; + +SELECT '-- multiple quotas in one command'; +CREATE QUOTA q1_01297, q2_01297 FOR 1 day MAX errors=5; +SHOW CREATE QUOTA q1_01297, q2_01297; +ALTER QUOTA q1_01297, q2_01297 FOR 1 day TRACKING ONLY TO r1_01297; +SHOW CREATE QUOTA q1_01297, q2_01297; +DROP QUOTA q1_01297, q2_01297; + +SELECT '-- system.quotas'; +CREATE QUOTA q1_01297 KEYED BY user_name TO r1_01297; +CREATE QUOTA q2_01297 FOR 2 MONTH MAX errors = 11, queries = 100, result_rows = 1000, result_bytes = 10000, read_rows = 1001, read_bytes = 10001, execution_time=2.5 TO r1_01297, u1_01297; +CREATE QUOTA q3_01297 KEYED BY client_key, user_name FOR 0.5 YEAR ERRORS MAX 11, QUERIES MAX 100, FOR 2 MONTH RESULT ROWS MAX 1002; +CREATE QUOTA q4_01297 FOR 1 WEEK TRACKING ONLY TO ALL EXCEPT u1_01297; +SELECT name, storage, keys, durations, apply_to_all, apply_to_list, apply_to_except FROM system.quotas WHERE name LIKE 'q%\_01297' ORDER BY name; + +SELECT '-- system.quota_limits'; +SELECT * FROM system.quota_limits WHERE quota_name LIKE 'q%\_01297' ORDER BY quota_name, duration; +DROP QUOTA q1_01297, q2_01297, q3_01297, q4_01297; + +DROP ROLE r1_01297; +DROP USER u1_01297; From 6146766465486724c1baace82c1bdf786d7ffbd6 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 6 Jun 2020 00:31:37 +0300 Subject: [PATCH 238/329] Use function ParserList::parseUtil() to parse lists more accurately. --- src/Parsers/ExpressionListParsers.cpp | 48 ++-- src/Parsers/ExpressionListParsers.h | 37 +++ src/Parsers/ParserCreateQuotaQuery.cpp | 91 ++++---- src/Parsers/ParserCreateRoleQuery.cpp | 16 +- src/Parsers/ParserCreateRowPolicyQuery.cpp | 76 +++---- .../ParserCreateSettingsProfileQuery.cpp | 18 +- src/Parsers/ParserCreateUserQuery.cpp | 210 +++++++++++------- src/Parsers/ParserGrantQuery.cpp | 66 ++++-- src/Parsers/ParserRolesOrUsersSet.cpp | 91 ++++---- src/Parsers/ParserRowPolicyName.cpp | 49 ++-- src/Parsers/ParserSettingsProfileElement.cpp | 160 ++++++++----- src/Parsers/ParserUserNameWithHost.cpp | 71 +++--- .../parseIdentifierOrStringLiteral.cpp | 53 +++-- 13 files changed, 578 insertions(+), 408 deletions(-) diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index a967ae19691..e33e80f1f18 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -76,41 +76,25 @@ const char * ParserTupleElementExpression::operators[] = bool ParserList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - bool first = true; + ASTs elements; + + auto parse_element = [&] + { + ASTPtr element; + if (!elem_parser->parse(pos, element, expected)) + return false; + + elements.push_back(element); + return true; + }; + + if (!parseUtil(pos, expected, parse_element, *separator_parser, allow_empty)) + return false; auto list = std::make_shared(result_separator); + list->children = std::move(elements); node = list; - - while (true) - { - if (first) - { - ASTPtr elem; - if (!elem_parser->parse(pos, elem, expected)) - break; - - list->children.push_back(elem); - first = false; - } - else - { - auto prev_pos = pos; - - if (!separator_parser->ignore(pos, expected)) - break; - - ASTPtr elem; - if (!elem_parser->parse(pos, elem, expected)) - { - pos = prev_pos; - break; - } - - list->children.push_back(elem); - } - } - - return allow_empty || !first; + return true; } diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index 0cef29b6d67..93a47648a0b 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -26,6 +26,43 @@ public: , result_separator(result_separator_) { } + + template + static bool parseUtil(Pos & pos, Expected & expected, const F & parse_element, IParser & separator_parser_, bool allow_empty_ = true) + { + Pos begin = pos; + if (!parse_element()) + { + pos = begin; + return allow_empty_; + } + + while (true) + { + begin = pos; + if (!separator_parser_.ignore(pos, expected) || !parse_element()) + { + pos = begin; + return true; + } + } + + return false; + } + + template + static bool parseUtil(Pos & pos, Expected & expected, const F & parse_element, TokenType separator, bool allow_empty_ = true) + { + ParserToken sep_parser{separator}; + return parseUtil(pos, expected, parse_element, sep_parser, allow_empty_); + } + + template + static bool parseUtil(Pos & pos, Expected & expected, const F & parse_element, bool allow_empty_ = true) + { + return parseUtil(pos, expected, parse_element, TokenType::Comma, allow_empty_); + } + protected: const char * getName() const override { return "list of elements"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; diff --git a/src/Parsers/ParserCreateQuotaQuery.cpp b/src/Parsers/ParserCreateQuotaQuery.cpp index f83bac975b0..324519b9c01 100644 --- a/src/Parsers/ParserCreateQuotaQuery.cpp +++ b/src/Parsers/ParserCreateQuotaQuery.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -126,17 +127,17 @@ namespace } - bool parseLimit(IParserBase::Pos & pos, Expected & expected, bool first, bool & max_prefix_encountered, ResourceType & resource_type, ResourceAmount & max) + bool parseLimits(IParserBase::Pos & pos, Expected & expected, std::vector> & limits) { - return IParserBase::wrapParseImpl(pos, [&] - { - if (!first && !ParserToken{TokenType::Comma}.ignore(pos, expected)) - return false; + std::vector> res_limits; + bool max_prefix_encountered = false; + auto parse_limit = [&] + { max_prefix_encountered |= ParserKeyword{"MAX"}.ignore(pos, expected); - ResourceType res_resource_type; - if (!parseResourceType(pos, expected, res_resource_type)) + ResourceType resource_type; + if (!parseResourceType(pos, expected, resource_type)) return false; if (max_prefix_encountered) @@ -149,25 +150,32 @@ namespace return false; } - ResourceAmount res_max; - if (!parseMaxAmount(pos, expected, res_resource_type, res_max)) + ResourceAmount max; + if (!parseMaxAmount(pos, expected, resource_type, max)) return false; - resource_type = res_resource_type; - max = res_max; + res_limits.emplace_back(resource_type, max); return true; - }); + }; + + if (!ParserList::parseUtil(pos, expected, parse_limit, false)) + return false; + + limits = std::move(res_limits); + return true; } - bool parseIntervalWithLimits(IParserBase::Pos & pos, Expected & expected, ASTCreateQuotaQuery::Limits & limits) + bool parseIntervalsWithLimits(IParserBase::Pos & pos, Expected & expected, std::vector & all_limits) { - return IParserBase::wrapParseImpl(pos, [&] + std::vector res_all_limits; + + auto parse_interval_with_limits = [&] { - ASTCreateQuotaQuery::Limits new_limits; if (!ParserKeyword{"FOR"}.ignore(pos, expected)) return false; - new_limits.randomize_interval = ParserKeyword{"RANDOMIZED"}.ignore(pos, expected); + ASTCreateQuotaQuery::Limits limits; + limits.randomize_interval = ParserKeyword{"RANDOMIZED"}.ignore(pos, expected); ParserKeyword{"INTERVAL"}.ignore(pos, expected); @@ -181,53 +189,34 @@ namespace if (!parseIntervalKind(pos, expected, interval_kind)) return false; - new_limits.duration = std::chrono::seconds(static_cast(num_intervals * interval_kind.toAvgSeconds())); + limits.duration = std::chrono::seconds(static_cast(num_intervals * interval_kind.toAvgSeconds())); + std::vector> maxs; if (ParserKeyword{"NO LIMITS"}.ignore(pos, expected)) { - new_limits.drop = true; + limits.drop = true; } else if (ParserKeyword{"TRACKING ONLY"}.ignore(pos, expected)) { } + else if (parseLimits(pos, expected, maxs)) + { + for (const auto & [resource_type, max] : maxs) + limits.max[resource_type] = max; + } else - { - ResourceType resource_type; - ResourceAmount max; - bool max_prefix_encountered = false; - if (!parseLimit(pos, expected, true, max_prefix_encountered, resource_type, max)) - return false; + return false; - new_limits.max[resource_type] = max; - while (parseLimit(pos, expected, false, max_prefix_encountered, resource_type, max)) - new_limits.max[resource_type] = max; - } - - limits = new_limits; + res_all_limits.emplace_back(std::move(limits)); return true; - }); - } + }; - bool parseIntervalsWithLimits(IParserBase::Pos & pos, Expected & expected, std::vector & all_limits) - { - return IParserBase::wrapParseImpl(pos, [&] - { - size_t old_size = all_limits.size(); - do - { - ASTCreateQuotaQuery::Limits limits; - if (!parseIntervalWithLimits(pos, expected, limits)) - { - all_limits.resize(old_size); - return false; - } - all_limits.push_back(limits); - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); - return true; - }); - } + if (!ParserList::parseUtil(pos, expected, parse_interval_with_limits, false)) + return false; + all_limits = std::move(res_all_limits); + return true; + } bool parseToRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) { diff --git a/src/Parsers/ParserCreateRoleQuery.cpp b/src/Parsers/ParserCreateRoleQuery.cpp index 6feeefa4657..5863136750f 100644 --- a/src/Parsers/ParserCreateRoleQuery.cpp +++ b/src/Parsers/ParserCreateRoleQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -23,7 +24,7 @@ namespace }); } - bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & settings) + bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::vector> & settings) { return IParserBase::wrapParseImpl(pos, [&] { @@ -36,10 +37,7 @@ namespace if (!elements_p.parse(pos, new_settings_ast, expected)) return false; - if (!settings) - settings = std::make_shared(); - const auto & new_settings = new_settings_ast->as(); - settings->elements.insert(settings->elements.end(), new_settings.elements.begin(), new_settings.elements.end()); + settings = std::move(new_settings_ast->as().elements); return true; }); } @@ -99,8 +97,14 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (alter && new_name.empty() && (names.size() == 1) && parseRenameTo(pos, expected, new_name)) continue; - if (parseSettings(pos, expected, attach_mode, settings)) + std::vector> new_settings; + if (parseSettings(pos, expected, attach_mode, new_settings)) + { + if (!settings) + settings = std::make_shared(); + boost::range::push_back(settings->elements, std::move(new_settings)); continue; + } if (cluster.empty() && parseOnCluster(pos, expected, cluster)) continue; diff --git a/src/Parsers/ParserCreateRowPolicyQuery.cpp b/src/Parsers/ParserCreateRowPolicyQuery.cpp index c9fe15d391f..fae5bd35b43 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/ParserCreateRowPolicyQuery.cpp @@ -75,6 +75,7 @@ namespace }); } + void addAllCommands(boost::container::flat_set & commands) { for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) @@ -84,44 +85,47 @@ namespace } } - bool parseCommands(IParserBase::Pos & pos, Expected & expected, boost::container::flat_set & commands) + + bool parseCommands(IParserBase::Pos & pos, Expected & expected, + boost::container::flat_set & commands) { - return IParserBase::wrapParseImpl(pos, [&] + boost::container::flat_set res_commands; + + auto parse_command = [&] { if (ParserKeyword{"ALL"}.ignore(pos, expected)) { - addAllCommands(commands); + addAllCommands(res_commands); return true; } - boost::container::flat_set res_commands; - do + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) { - bool found_keyword = false; - for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + const std::string_view & command = ConditionTypeInfo::get(condition_type).command; + if (ParserKeyword{command.data()}.ignore(pos, expected)) { - const std::string_view & command = ConditionTypeInfo::get(condition_type).command; - if (ParserKeyword{command.data()}.ignore(pos, expected)) - { - res_commands.emplace(command); - found_keyword = true; - break; - } + res_commands.emplace(command); + return true; } - - if (!found_keyword) - return false; } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); - commands = std::move(res_commands); - return true; - }); + return false; + }; + + if (!ParserList::parseUtil(pos, expected, parse_command, false)) + return false; + + commands = std::move(res_commands); + return true; } - bool parseForClause(IParserBase::Pos & pos, Expected & expected, bool alter, std::vector> & conditions) + + bool + parseForClauses(IParserBase::Pos & pos, Expected & expected, bool alter, std::vector> & conditions) { - return IParserBase::wrapParseImpl(pos, [&] + std::vector> res_conditions; + + auto parse_for_clause = [&] { boost::container::flat_set commands; @@ -158,32 +162,20 @@ namespace if (commands.count(type_info.command)) { if (type_info.is_check && check) - conditions.emplace_back(condition_type, *check); + res_conditions.emplace_back(condition_type, *check); else if (filter) - conditions.emplace_back(condition_type, *filter); + res_conditions.emplace_back(condition_type, *filter); } } return true; - }); - } + }; - bool parseForClauses( - IParserBase::Pos & pos, Expected & expected, bool alter, std::vector> & conditions) - { - return IParserBase::wrapParseImpl(pos, [&] - { - std::vector> res_conditions; - do - { - if (!parseForClause(pos, expected, alter, res_conditions)) - return false; - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + if (!ParserList::parseUtil(pos, expected, parse_for_clause, false)) + return false; - conditions = std::move(res_conditions); - return true; - }); + conditions = std::move(res_conditions); + return true; } bool parseToRoles(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & roles) diff --git a/src/Parsers/ParserCreateSettingsProfileQuery.cpp b/src/Parsers/ParserCreateSettingsProfileQuery.cpp index 56bd39b9230..797379509e4 100644 --- a/src/Parsers/ParserCreateSettingsProfileQuery.cpp +++ b/src/Parsers/ParserCreateSettingsProfileQuery.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB @@ -25,7 +26,7 @@ namespace }); } - bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & settings) + bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::vector> & settings) { return IParserBase::wrapParseImpl(pos, [&] { @@ -38,10 +39,7 @@ namespace if (!elements_p.parse(pos, new_settings_ast, expected)) return false; - if (!settings) - settings = std::make_shared(); - const auto & new_settings = new_settings_ast->as(); - settings->elements.insert(settings->elements.end(), new_settings.elements.begin(), new_settings.elements.end()); + settings = std::move(new_settings_ast->as().elements); return true; }); } @@ -51,7 +49,7 @@ namespace return IParserBase::wrapParseImpl(pos, [&] { ASTPtr ast; - if (roles || !ParserKeyword{"TO"}.ignore(pos, expected)) + if (!ParserKeyword{"TO"}.ignore(pos, expected)) return false; ParserRolesOrUsersSet roles_p; @@ -119,8 +117,14 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec if (alter && new_name.empty() && (names.size() == 1) && parseRenameTo(pos, expected, new_name)) continue; - if (parseSettings(pos, expected, attach_mode, settings)) + std::vector> new_settings; + if (parseSettings(pos, expected, attach_mode, new_settings)) + { + if (!settings) + settings = std::make_shared(); + boost::range::push_back(settings->elements, std::move(new_settings)); continue; + } if (cluster.empty() && parseOnCluster(pos, expected, cluster)) continue; diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index ff7f2dc8790..4641c94d592 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace DB @@ -32,7 +33,7 @@ namespace } - bool parseAuthentication(IParserBase::Pos & pos, Expected & expected, std::optional & authentication) + bool parseAuthentication(IParserBase::Pos & pos, Expected & expected, Authentication & authentication) { return IParserBase::wrapParseImpl(pos, [&] { @@ -96,91 +97,109 @@ namespace authentication = Authentication{*type}; if (expect_password) - authentication->setPassword(password); + authentication.setPassword(password); else if (expect_hash) - authentication->setPasswordHashHex(password); + authentication.setPasswordHashHex(password); return true; }); } - bool parseHosts(IParserBase::Pos & pos, Expected & expected, const char * prefix, std::optional & hosts) + bool parseHostsWithoutPrefix(IParserBase::Pos & pos, Expected & expected, AllowedClientHosts & hosts) + { + AllowedClientHosts res_hosts; + + auto parse_host = [&] + { + if (ParserKeyword{"NONE"}.ignore(pos, expected)) + return true; + + if (ParserKeyword{"ANY"}.ignore(pos, expected)) + { + res_hosts.addAnyHost(); + return true; + } + + if (ParserKeyword{"LOCAL"}.ignore(pos, expected)) + { + res_hosts.addLocalHost(); + return true; + } + + if (ParserKeyword{"REGEXP"}.ignore(pos, expected)) + { + ASTPtr ast; + if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) + return false; + + for (const auto & name_regexp_ast : ast->children) + res_hosts.addNameRegexp(name_regexp_ast->as().value.safeGet()); + return true; + } + + if (ParserKeyword{"NAME"}.ignore(pos, expected)) + { + ASTPtr ast; + if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) + return false; + + for (const auto & name_ast : ast->children) + res_hosts.addName(name_ast->as().value.safeGet()); + + return true; + } + + if (ParserKeyword{"IP"}.ignore(pos, expected)) + { + ASTPtr ast; + if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) + return false; + + for (const auto & subnet_ast : ast->children) + res_hosts.addSubnet(subnet_ast->as().value.safeGet()); + + return true; + } + + if (ParserKeyword{"LIKE"}.ignore(pos, expected)) + { + ASTPtr ast; + if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) + return false; + + for (const auto & pattern_ast : ast->children) + res_hosts.addLikePattern(pattern_ast->as().value.safeGet()); + + return true; + } + + return false; + }; + + if (!ParserList::parseUtil(pos, expected, parse_host, false)) + return false; + + hosts = std::move(res_hosts); + return true; + } + + + bool parseHosts(IParserBase::Pos & pos, Expected & expected, const String & prefix, AllowedClientHosts & hosts) { return IParserBase::wrapParseImpl(pos, [&] { - if (prefix && !ParserKeyword{prefix}.ignore(pos, expected)) + if (!prefix.empty() && !ParserKeyword{prefix.c_str()}.ignore(pos, expected)) return false; if (!ParserKeyword{"HOST"}.ignore(pos, expected)) return false; - if (ParserKeyword{"ANY"}.ignore(pos, expected)) - { - if (!hosts) - hosts.emplace(); - hosts->addAnyHost(); - return true; - } + AllowedClientHosts res_hosts; + if (!parseHostsWithoutPrefix(pos, expected, res_hosts)) + return false; - if (ParserKeyword{"NONE"}.ignore(pos, expected)) - { - if (!hosts) - hosts.emplace(); - return true; - } - - AllowedClientHosts new_hosts; - do - { - if (ParserKeyword{"LOCAL"}.ignore(pos, expected)) - { - new_hosts.addLocalHost(); - } - else if (ParserKeyword{"REGEXP"}.ignore(pos, expected)) - { - ASTPtr ast; - if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) - return false; - - for (const auto & name_regexp_ast : ast->children) - new_hosts.addNameRegexp(name_regexp_ast->as().value.safeGet()); - } - else if (ParserKeyword{"NAME"}.ignore(pos, expected)) - { - ASTPtr ast; - if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) - return false; - - for (const auto & name_ast : ast->children) - new_hosts.addName(name_ast->as().value.safeGet()); - } - else if (ParserKeyword{"IP"}.ignore(pos, expected)) - { - ASTPtr ast; - if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) - return false; - - for (const auto & subnet_ast : ast->children) - new_hosts.addSubnet(subnet_ast->as().value.safeGet()); - } - else if (ParserKeyword{"LIKE"}.ignore(pos, expected)) - { - ASTPtr ast; - if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) - return false; - - for (const auto & pattern_ast : ast->children) - new_hosts.addLikePattern(pattern_ast->as().value.safeGet()); - } - else - return false; - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); - - if (!hosts) - hosts.emplace(); - hosts->add(new_hosts); + hosts.add(std::move(res_hosts)); return true; }); } @@ -206,7 +225,7 @@ namespace } - bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & settings) + bool parseSettings(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::vector> & settings) { return IParserBase::wrapParseImpl(pos, [&] { @@ -215,14 +234,11 @@ namespace ASTPtr new_settings_ast; ParserSettingsProfileElements elements_p; - elements_p.useInheritKeyword(true).useIDMode(id_mode); + elements_p.useIDMode(id_mode); if (!elements_p.parse(pos, new_settings_ast, expected)) return false; - if (!settings) - settings = std::make_shared(); - const auto & new_settings = new_settings_ast->as(); - settings->elements.insert(settings->elements.end(), new_settings.elements.begin(), new_settings.elements.end()); + settings = std::move(new_settings_ast->as().elements); return true; }); } @@ -286,14 +302,33 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec while (true) { - if (!authentication && parseAuthentication(pos, expected, authentication)) - continue; + if (!authentication) + { + Authentication new_authentication; + if (parseAuthentication(pos, expected, new_authentication)) + { + authentication = std::move(new_authentication); + continue; + } + } - if (parseHosts(pos, expected, nullptr, hosts)) + AllowedClientHosts new_hosts; + if (parseHosts(pos, expected, "", new_hosts)) + { + if (!hosts) + hosts.emplace(); + hosts->add(new_hosts); continue; + } - if (parseSettings(pos, expected, attach_mode, settings)) + std::vector> new_settings; + if (parseSettings(pos, expected, attach_mode, new_settings)) + { + if (!settings) + settings = std::make_shared(); + boost::range::push_back(settings->elements, std::move(new_settings)); continue; + } if (!default_roles && parseDefaultRoles(pos, expected, attach_mode, default_roles)) continue; @@ -306,8 +341,21 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (new_name.empty() && (names->size() == 1) && parseRenameTo(pos, expected, new_name)) continue; - if (parseHosts(pos, expected, "ADD", add_hosts) || parseHosts(pos, expected, "DROP", remove_hosts)) + if (parseHosts(pos, expected, "ADD", new_hosts)) + { + if (!add_hosts) + add_hosts.emplace(); + add_hosts->add(new_hosts); continue; + } + + if (parseHosts(pos, expected, "DROP", new_hosts)) + { + if (!remove_hosts) + remove_hosts.emplace(); + remove_hosts->add(new_hosts); + continue; + } } break; diff --git a/src/Parsers/ParserGrantQuery.cpp b/src/Parsers/ParserGrantQuery.cpp index 03c0daa08a3..5eb5353f2ee 100644 --- a/src/Parsers/ParserGrantQuery.cpp +++ b/src/Parsers/ParserGrantQuery.cpp @@ -2,8 +2,11 @@ #include #include #include +#include +#include #include #include +#include #include #include @@ -66,15 +69,13 @@ namespace if (!ParserToken{TokenType::OpeningRoundBracket}.ignore(pos, expected)) return false; + ASTPtr ast; + if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) + return false; + Strings res_columns; - do - { - ASTPtr column_ast; - if (!ParserIdentifier().parse(pos, column_ast, expected)) - return false; - res_columns.emplace_back(getIdentifierName(column_ast)); - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + for (const auto & child : ast->children) + res_columns.emplace_back(getIdentifierName(child)); if (!ParserToken{TokenType::ClosingRoundBracket}.ignore(pos, expected)) return false; @@ -150,25 +151,42 @@ namespace } + bool parseAccessTypesWithColumns(IParser::Pos & pos, Expected & expected, + std::vector> & access_and_columns) + { + std::vector> res; + + auto parse_access_and_columns = [&] + { + AccessFlags access_flags; + if (!parseAccessFlags(pos, expected, access_flags)) + return false; + + Strings columns; + parseColumnNames(pos, expected, columns); + res.emplace_back(access_flags, std::move(columns)); + return true; + }; + + if (!ParserList::parseUtil(pos, expected, parse_access_and_columns, false)) + return false; + + access_and_columns = std::move(res); + return true; + } + + bool parseAccessRightsElements(IParser::Pos & pos, Expected & expected, AccessRightsElements & elements) { return IParserBase::wrapParseImpl(pos, [&] { AccessRightsElements res_elements; - do + + auto parse_around_on = [&] { std::vector> access_and_columns; - do - { - AccessFlags access_flags; - if (!parseAccessFlags(pos, expected, access_flags)) - return false; - - Strings columns; - parseColumnNames(pos, expected, columns); - access_and_columns.emplace_back(access_flags, std::move(columns)); - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + if (!parseAccessTypesWithColumns(pos, expected, access_and_columns)) + return false; if (!ParserKeyword{"ON"}.ignore(pos, expected)) return false; @@ -190,8 +208,12 @@ namespace element.table = table_name; res_elements.emplace_back(std::move(element)); } - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + + return true; + }; + + if (!ParserList::parseUtil(pos, expected, parse_around_on, false)) + return false; elements = std::move(res_elements); return true; diff --git a/src/Parsers/ParserRolesOrUsersSet.cpp b/src/Parsers/ParserRolesOrUsersSet.cpp index 1ba2c05f671..0f3ba3f0f84 100644 --- a/src/Parsers/ParserRolesOrUsersSet.cpp +++ b/src/Parsers/ParserRolesOrUsersSet.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -11,11 +12,15 @@ namespace DB { namespace { - bool parseRoleNameOrID(IParserBase::Pos & pos, Expected & expected, bool parse_id, String & res) + bool parseRoleNameOrID( + IParserBase::Pos & pos, + Expected & expected, + bool id_mode, + String & res) { return IParserBase::wrapParseImpl(pos, [&] { - if (!parse_id) + if (!id_mode) return parseRoleName(pos, expected, res); if (!ParserKeyword{"ID"}.ignore(pos, expected)) @@ -40,60 +45,56 @@ namespace Expected & expected, bool id_mode, bool allow_all, - bool allow_current_user_tag, + bool allow_current_user, Strings & names, bool & all, bool & current_user) { - return IParserBase::wrapParseImpl(pos, [&] - { - bool res_all = false; - bool res_current_user = false; - Strings res_names; - while (true) - { - if (ParserKeyword{"NONE"}.ignore(pos, expected)) - { - } - else if ( - allow_current_user_tag - && (ParserKeyword{"CURRENT_USER"}.ignore(pos, expected) || ParserKeyword{"currentUser"}.ignore(pos, expected))) - { - if (ParserToken{TokenType::OpeningRoundBracket}.ignore(pos, expected)) - { - if (!ParserToken{TokenType::ClosingRoundBracket}.ignore(pos, expected)) - return false; - } - res_current_user = true; - } - else if (allow_all && ParserKeyword{"ALL"}.ignore(pos, expected)) - { - res_all = true; - } - else - { - String name; - if (!parseRoleNameOrID(pos, expected, id_mode, name)) - return false; - res_names.push_back(name); - } + bool res_all = false; + bool res_current_user = false; + Strings res_names; - if (!ParserToken{TokenType::Comma}.ignore(pos, expected)) - break; + auto parse_element = [&] + { + if (ParserKeyword{"NONE"}.ignore(pos, expected)) + return true; + + if (allow_all && ParserKeyword{"ALL"}.ignore(pos, expected)) + { + res_all = true; + return true; } - all = res_all; - current_user = res_current_user; - names = std::move(res_names); - return true; - }); + if (allow_current_user && parseCurrentUserTag(pos, expected)) + { + res_current_user = true; + return true; + } + + String name; + if (parseRoleNameOrID(pos, expected, id_mode, name)) + { + res_names.emplace_back(std::move(name)); + return true; + } + + return false; + }; + + if (!ParserList::parseUtil(pos, expected, parse_element, false)) + return false; + + names = std::move(res_names); + all = res_all; + current_user = res_current_user; + return true; } bool parseExceptAndAfterExcept( IParserBase::Pos & pos, Expected & expected, bool id_mode, - bool allow_current_user_tag, + bool allow_current_user, Strings & except_names, bool & except_current_user) { @@ -102,8 +103,8 @@ namespace if (!ParserKeyword{"EXCEPT"}.ignore(pos, expected)) return false; - bool dummy; - return parseBeforeExcept(pos, expected, id_mode, false, allow_current_user_tag, except_names, dummy, except_current_user); + bool unused; + return parseBeforeExcept(pos, expected, id_mode, false, allow_current_user, except_names, unused, except_current_user); }); } } diff --git a/src/Parsers/ParserRowPolicyName.cpp b/src/Parsers/ParserRowPolicyName.cpp index a74132cdaca..a3e12009c9a 100644 --- a/src/Parsers/ParserRowPolicyName.cpp +++ b/src/Parsers/ParserRowPolicyName.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -67,21 +68,18 @@ namespace return false; std::vector> res; - std::optional pos_before_comma; - do + + auto parse_db_and_table_name = [&] { String database, table_name; if (!parseDBAndTableName(pos, expected, database, table_name)) - { - if (!pos_before_comma) - return false; - pos = *pos_before_comma; - break; - } + return false; res.emplace_back(std::move(database), std::move(table_name)); - pos_before_comma = pos; - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + return true; + }; + + if (!ParserList::parseUtil(pos, expected, parse_db_and_table_name, false)) + return false; database_and_table_names = std::move(res); return true; @@ -165,21 +163,28 @@ bool ParserRowPolicyName::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte bool ParserRowPolicyNames::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { std::vector name_parts; + size_t num_added_names_last_time = 0; String cluster; - do + auto parse_around_on = [&] { - std::vector new_name_parts; - if (!parseRowPolicyNamesAroundON(pos, expected, name_parts.empty(), name_parts.empty(), allow_on_cluster, new_name_parts, cluster)) - return false; + if (!name_parts.empty()) + { + if ((num_added_names_last_time != 1) || !cluster.empty()) + return false; + } - size_t num_new_name_parts = new_name_parts.size(); - assert(num_new_name_parts >= 1); - boost::range::push_back(name_parts, std::move(new_name_parts)); - if ((num_new_name_parts != 1) || !cluster.empty()) - break; - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + std::vector new_name_parts; + if (!parseRowPolicyNamesAroundON(pos, expected, name_parts.empty(), name_parts.empty(), allow_on_cluster, new_name_parts, cluster)) + return false; + + num_added_names_last_time = new_name_parts.size(); + boost::range::push_back(name_parts, std::move(new_name_parts)); + return true; + }; + + if (!ParserList::parseUtil(pos, expected, parse_around_on, false)) + return false; auto result = std::make_shared(); result->name_parts = std::move(name_parts); diff --git a/src/Parsers/ParserSettingsProfileElement.cpp b/src/Parsers/ParserSettingsProfileElement.cpp index 2dd65e6ae7b..39e1f2d3594 100644 --- a/src/Parsers/ParserSettingsProfileElement.cpp +++ b/src/Parsers/ParserSettingsProfileElement.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -11,12 +12,19 @@ namespace DB { namespace { - bool parseProfileNameOrID(IParserBase::Pos & pos, Expected & expected, bool parse_id, String & res) + bool parseProfileKeyword(IParserBase::Pos & pos, Expected & expected, bool use_inherit_keyword) + { + return ParserKeyword{"PROFILE"}.ignore(pos, expected) || + (use_inherit_keyword && ParserKeyword{"INHERIT"}.ignore(pos, expected)); + } + + + bool parseProfileNameOrID(IParserBase::Pos & pos, Expected & expected, bool id_mode, String & res) { return IParserBase::wrapParseImpl(pos, [&] { ASTPtr ast; - if (!parse_id) + if (!id_mode) return parseIdentifierOrStringLiteral(pos, expected, res); if (!ParserKeyword{"ID"}.ignore(pos, expected)) @@ -96,52 +104,98 @@ namespace return false; }); } + + + bool parseSettingNameWithValueOrConstraints( + IParserBase::Pos & pos, + Expected & expected, + String & setting_name, + Field & value, + Field & min_value, + Field & max_value, + std::optional & readonly) + { + return IParserBase::wrapParseImpl(pos, [&] + { + ASTPtr name_ast; + if (!ParserIdentifier{}.parse(pos, name_ast, expected)) + return false; + + String res_setting_name = getIdentifierName(name_ast); + Field res_value; + Field res_min_value; + Field res_max_value; + std::optional res_readonly; + + bool has_value_or_constraint = false; + while (parseValue(pos, expected, res_value) || parseMinMaxValue(pos, expected, res_min_value, res_max_value) + || parseReadonlyOrWritableKeyword(pos, expected, res_readonly)) + { + has_value_or_constraint = true; + } + + if (!has_value_or_constraint) + return false; + + setting_name = std::move(res_setting_name); + value = std::move(res_value); + min_value = std::move(res_min_value); + max_value = std::move(res_max_value); + readonly = res_readonly; + return true; + }); + } + + + bool parseSettingsProfileElement(IParserBase::Pos & pos, + Expected & expected, + bool id_mode, + bool use_inherit_keyword, + bool previous_element_was_parent_profile, + std::shared_ptr & result) + { + return IParserBase::wrapParseImpl(pos, [&] + { + String parent_profile; + String setting_name; + Field value; + Field min_value; + Field max_value; + std::optional readonly; + + if (parseSettingNameWithValueOrConstraints(pos, expected, setting_name, value, min_value, max_value, readonly)) + { + } + else if (parseProfileKeyword(pos, expected, use_inherit_keyword) || previous_element_was_parent_profile) + { + if (!parseProfileNameOrID(pos, expected, id_mode, parent_profile)) + return false; + } + else + return false; + + result = std::make_shared(); + result->parent_profile = std::move(parent_profile); + result->setting_name = std::move(setting_name); + result->value = std::move(value); + result->min_value = std::move(min_value); + result->max_value = std::move(max_value); + result->readonly = readonly; + result->id_mode = id_mode; + result->use_inherit_keyword = use_inherit_keyword; + return true; + }); + } } bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - String parent_profile; - String setting_name; - Field value; - Field min_value; - Field max_value; - std::optional readonly; + std::shared_ptr res; + if (!parseSettingsProfileElement(pos, expected, id_mode, use_inherit_keyword, false, res)) + return false; - if (ParserKeyword{"PROFILE"}.ignore(pos, expected) || - (use_inherit_keyword && ParserKeyword{"INHERIT"}.ignore(pos, expected))) - { - if (!parseProfileNameOrID(pos, expected, id_mode, parent_profile)) - return false; - } - else - { - ASTPtr name_ast; - if (!ParserIdentifier{}.parse(pos, name_ast, expected)) - return false; - setting_name = getIdentifierName(name_ast); - - bool has_value_or_constraint = false; - while (parseValue(pos, expected, value) || parseMinMaxValue(pos, expected, min_value, max_value) - || parseReadonlyOrWritableKeyword(pos, expected, readonly)) - { - has_value_or_constraint = true; - } - - if (!has_value_or_constraint) - return false; - } - - auto result = std::make_shared(); - result->parent_profile = std::move(parent_profile); - result->setting_name = std::move(setting_name); - result->value = std::move(value); - result->min_value = std::move(min_value); - result->max_value = std::move(max_value); - result->readonly = readonly; - result->id_mode = id_mode; - result->use_inherit_keyword = use_inherit_keyword; - node = result; + node = res; return true; } @@ -155,17 +209,21 @@ bool ParserSettingsProfileElements::parseImpl(Pos & pos, ASTPtr & node, Expected } else { - ParserSettingsProfileElement element_p; - element_p.useIDMode(id_mode).useInheritKeyword(use_inherit_keyword); - do + bool previous_element_was_parent_profile = false; + + auto parse_element = [&] { - ASTPtr ast; - if (!element_p.parse(pos, ast, expected)) + std::shared_ptr element; + if (!parseSettingsProfileElement(pos, expected, id_mode, use_inherit_keyword, previous_element_was_parent_profile, element)) return false; - auto element = typeid_cast>(ast); - elements.push_back(std::move(element)); - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + + elements.push_back(element); + previous_element_was_parent_profile = !element->parent_profile.empty(); + return true; + }; + + if (!ParserList::parseUtil(pos, expected, parse_element, false)) + return false; } auto result = std::make_shared(); diff --git a/src/Parsers/ParserUserNameWithHost.cpp b/src/Parsers/ParserUserNameWithHost.cpp index 19ec7a9bbd1..9cb4bb6fc97 100644 --- a/src/Parsers/ParserUserNameWithHost.cpp +++ b/src/Parsers/ParserUserNameWithHost.cpp @@ -1,35 +1,52 @@ #include #include #include +#include #include #include namespace DB { +namespace +{ + bool parseUserNameWithHost(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & ast) + { + return IParserBase::wrapParseImpl(pos, [&] + { + String base_name; + if (!parseIdentifierOrStringLiteral(pos, expected, base_name)) + return false; + + boost::algorithm::trim(base_name); + + String host_pattern; + if (ParserToken{TokenType::At}.ignore(pos, expected)) + { + if (!parseIdentifierOrStringLiteral(pos, expected, host_pattern)) + return false; + + boost::algorithm::trim(host_pattern); + if (host_pattern == "%") + host_pattern.clear(); + } + + ast = std::make_shared(); + ast->base_name = std::move(base_name); + ast->host_pattern = std::move(host_pattern); + return true; + }); + } +} + + bool ParserUserNameWithHost::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - String base_name; - if (!parseIdentifierOrStringLiteral(pos, expected, base_name)) + std::shared_ptr res; + if (!parseUserNameWithHost(pos, expected, res)) return false; - boost::algorithm::trim(base_name); - - String host_pattern; - if (ParserToken{TokenType::At}.ignore(pos, expected)) - { - if (!parseIdentifierOrStringLiteral(pos, expected, host_pattern)) - return false; - - boost::algorithm::trim(host_pattern); - if (host_pattern == "%") - host_pattern.clear(); - } - - auto result = std::make_shared(); - result->base_name = std::move(base_name); - result->host_pattern = std::move(host_pattern); - node = result; + node = res; return true; } @@ -37,15 +54,19 @@ bool ParserUserNameWithHost::parseImpl(Pos & pos, ASTPtr & node, Expected & expe bool ParserUserNamesWithHost::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { std::vector> names; - do + + auto parse_single_name = [&] { - ASTPtr ast; - if (!ParserUserNameWithHost{}.parse(pos, ast, expected)) + std::shared_ptr ast; + if (!parseUserNameWithHost(pos, expected, ast)) return false; - names.emplace_back(typeid_cast>(ast)); - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); + names.emplace_back(std::move(ast)); + return true; + }; + + if (!ParserList::parseUtil(pos, expected, parse_single_name, false)) + return false; auto result = std::make_shared(); result->names = std::move(names); diff --git a/src/Parsers/parseIdentifierOrStringLiteral.cpp b/src/Parsers/parseIdentifierOrStringLiteral.cpp index 22c77af0b09..e1444eb2d5a 100644 --- a/src/Parsers/parseIdentifierOrStringLiteral.cpp +++ b/src/Parsers/parseIdentifierOrStringLiteral.cpp @@ -4,47 +4,52 @@ #include "ASTLiteral.h" #include "ASTIdentifier.h" #include +#include #include namespace DB { - bool parseIdentifierOrStringLiteral(IParser::Pos & pos, Expected & expected, String & result) { - ASTPtr res; - - if (!ParserIdentifier().parse(pos, res, expected)) + return IParserBase::wrapParseImpl(pos, [&] { - if (!ParserStringLiteral().parse(pos, res, expected)) - return false; + ASTPtr ast; + if (ParserIdentifier().parse(pos, ast, expected)) + { + result = getIdentifierName(ast); + return true; + } - result = res->as().value.safeGet(); - } - else - result = getIdentifierName(res); + if (ParserStringLiteral().parse(pos, ast, expected)) + { + result = ast->as().value.safeGet(); + return true; + } - return true; + return false; + }); } bool parseIdentifiersOrStringLiterals(IParser::Pos & pos, Expected & expected, Strings & result) { - return IParserBase::wrapParseImpl(pos, [&] + Strings res; + + auto parse_single_id_or_literal = [&] { - Strings strs; - do - { - String str; - if (!parseIdentifierOrStringLiteral(pos, expected, str)) - return false; + String str; + if (!parseIdentifierOrStringLiteral(pos, expected, str)) + return false; - strs.push_back(std::move(str)); - } - while (ParserToken{TokenType::Comma}.ignore(pos, expected)); - - result = std::move(strs); + res.emplace_back(std::move(str)); return true; - }); + }; + + if (!ParserList::parseUtil(pos, expected, parse_single_id_or_literal, false)) + return false; + + result = std::move(res); + return true; } } From 4187edd9f958b95b37b7a048d119691c4f275411 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 15 Jun 2020 01:07:52 +0300 Subject: [PATCH 239/329] Split integration test 'test_grant_and_revoke' into two tests. --- .../__init__.py | 0 .../test_create_user_and_login/test.py | 85 +++++++++++++++++++ tests/integration/test_role/__init__.py | 0 .../test.py | 40 +-------- 4 files changed, 86 insertions(+), 39 deletions(-) rename tests/integration/{test_grant_and_revoke => test_create_user_and_login}/__init__.py (100%) create mode 100644 tests/integration/test_create_user_and_login/test.py create mode 100644 tests/integration/test_role/__init__.py rename tests/integration/{test_grant_and_revoke => test_role}/test.py (77%) diff --git a/tests/integration/test_grant_and_revoke/__init__.py b/tests/integration/test_create_user_and_login/__init__.py similarity index 100% rename from tests/integration/test_grant_and_revoke/__init__.py rename to tests/integration/test_create_user_and_login/__init__.py diff --git a/tests/integration/test_create_user_and_login/test.py b/tests/integration/test_create_user_and_login/test.py new file mode 100644 index 00000000000..32bf0af6bb6 --- /dev/null +++ b/tests/integration/test_create_user_and_login/test.py @@ -0,0 +1,85 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV +import re + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance') + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + instance.query("CREATE TABLE test_table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()") + instance.query("INSERT INTO test_table VALUES (1,5), (2,10)") + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def cleanup_after_test(): + try: + yield + finally: + instance.query("DROP USER IF EXISTS A, B") + + +def test_login(): + instance.query("CREATE USER A") + instance.query("CREATE USER B") + assert instance.query("SELECT 1", user='A') == "1\n" + assert instance.query("SELECT 1", user='B') == "1\n" + + +def test_grant_and_revoke(): + instance.query("CREATE USER A") + assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') + + instance.query('GRANT SELECT ON test_table TO A') + assert instance.query("SELECT * FROM test_table", user='A') == "1\t5\n2\t10\n" + + instance.query('REVOKE SELECT ON test_table FROM A') + assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') + + +def test_grant_option(): + instance.query("CREATE USER A") + instance.query("CREATE USER B") + + instance.query('GRANT SELECT ON test_table TO A') + assert instance.query("SELECT * FROM test_table", user='A') == "1\t5\n2\t10\n" + assert "Not enough privileges" in instance.query_and_get_error("GRANT SELECT ON test_table TO B", user='A') + + instance.query('GRANT SELECT ON test_table TO A WITH GRANT OPTION') + instance.query("GRANT SELECT ON test_table TO B", user='A') + assert instance.query("SELECT * FROM test_table", user='B') == "1\t5\n2\t10\n" + + instance.query('REVOKE SELECT ON test_table FROM A, B') + + +def test_introspection(): + instance.query("CREATE USER A") + instance.query("CREATE USER B") + instance.query('GRANT SELECT ON test.table TO A') + instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION') + + assert instance.query("SHOW USERS") == TSV([ "A", "B", "default" ]) + assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A" ]) + assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) + + assert instance.query("SHOW GRANTS", user='A') == TSV([ "GRANT SELECT ON test.table TO A" ]) + assert instance.query("SHOW GRANTS", user='B') == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) + + assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ + TSV([[ "A", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], + [ "B", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) + + assert instance.query("SELECT * from system.grants WHERE user_name IN ('A', 'B') ORDER BY user_name, access_type, grant_option") ==\ + TSV([[ "A", "\N", "SELECT", "test", "table", "\N", 0, 0 ], + [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 0 ], + [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 1 ]]) diff --git a/tests/integration/test_role/__init__.py b/tests/integration/test_role/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_role/test.py similarity index 77% rename from tests/integration/test_grant_and_revoke/test.py rename to tests/integration/test_role/test.py index 7054ce28e59..92e9f00d326 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_role/test.py @@ -22,7 +22,7 @@ def started_cluster(): @pytest.fixture(autouse=True) -def reset_users_and_roles(): +def cleanup_after_test(): try: yield finally: @@ -30,39 +30,6 @@ def reset_users_and_roles(): instance.query("DROP ROLE IF EXISTS R1, R2") -def test_login(): - instance.query("CREATE USER A") - instance.query("CREATE USER B") - assert instance.query("SELECT 1", user='A') == "1\n" - assert instance.query("SELECT 1", user='B') == "1\n" - - -def test_grant_and_revoke(): - instance.query("CREATE USER A") - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') - - instance.query('GRANT SELECT ON test_table TO A') - assert instance.query("SELECT * FROM test_table", user='A') == "1\t5\n2\t10\n" - - instance.query('REVOKE SELECT ON test_table FROM A') - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') - - -def test_grant_option(): - instance.query("CREATE USER A") - instance.query("CREATE USER B") - - instance.query('GRANT SELECT ON test_table TO A') - assert instance.query("SELECT * FROM test_table", user='A') == "1\t5\n2\t10\n" - assert "Not enough privileges" in instance.query_and_get_error("GRANT SELECT ON test_table TO B", user='A') - - instance.query('GRANT SELECT ON test_table TO A WITH GRANT OPTION') - instance.query("GRANT SELECT ON test_table TO B", user='A') - assert instance.query("SELECT * FROM test_table", user='B') == "1\t5\n2\t10\n" - - instance.query('REVOKE SELECT ON test_table FROM A, B') - - def test_create_role(): instance.query("CREATE USER A") instance.query('CREATE ROLE R1') @@ -141,7 +108,6 @@ def test_introspection(): instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION') instance.query('REVOKE SELECT(x) ON test.table FROM R2') - assert instance.query("SHOW USERS") == TSV([ "A", "B", "default" ]) assert instance.query("SHOW ROLES") == TSV([ "R1", "R2" ]) assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT R1 TO A" ]) assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION" ]) @@ -155,10 +121,6 @@ def test_introspection(): assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([[ "R1", 0, 1, 1 ]]) assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([[ "R2", 1, 1, 1 ]]) - assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ - TSV([[ "A", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], - [ "B", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) - assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") ==\ TSV([[ "R1", "disk" ], [ "R2", "disk" ]]) From 9fe47df2e8bf2cc2535e2515d4a35cad09c36ae3 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 6 Jun 2020 10:21:02 +0300 Subject: [PATCH 240/329] Support multiple users/roles in SHOW CREATE USER(ROLE, etc.) and SHOW GRANTS FOR commands. Support syntax "SHOW CREATE USER ALL" and "SHOW GRANTS FOR ALL". --- src/Access/DiskAccessStorage.cpp | 4 +- src/Access/IAccessEntity.h | 49 +++-- .../InterpreterShowAccessEntitiesQuery.cpp | 21 ++- ...InterpreterShowCreateAccessEntityQuery.cpp | 94 +++++++--- .../InterpreterShowCreateAccessEntityQuery.h | 7 +- .../InterpreterShowGrantsQuery.cpp | 46 +++-- src/Interpreters/InterpreterShowGrantsQuery.h | 6 +- src/Parsers/ASTShowAccessEntitiesQuery.cpp | 58 +++--- src/Parsers/ASTShowAccessEntitiesQuery.h | 19 +- .../ASTShowCreateAccessEntityQuery.cpp | 45 ++++- src/Parsers/ASTShowCreateAccessEntityQuery.h | 19 +- src/Parsers/ASTShowGrantsQuery.cpp | 14 +- src/Parsers/ASTShowGrantsQuery.h | 5 +- src/Parsers/ParserDropAccessEntityQuery.cpp | 32 ++-- src/Parsers/ParserGrantQuery.cpp | 71 +------ src/Parsers/ParserShowAccessEntitiesQuery.cpp | 77 +++++--- src/Parsers/ParserShowAccessEntitiesQuery.h | 10 +- .../ParserShowCreateAccessEntityQuery.cpp | 174 +++++++++++++----- .../ParserShowCreateAccessEntityQuery.h | 9 + src/Parsers/ParserShowGrantsQuery.cpp | 23 ++- src/Parsers/parseDatabaseAndTableName.cpp | 70 +++++++ src/Parsers/parseDatabaseAndTableName.h | 5 +- .../test_create_user_and_login/test.py | 8 + tests/integration/test_quota/test.py | 4 +- tests/integration/test_role/test.py | 5 + tests/integration/test_row_policy/test.py | 12 +- .../integration/test_settings_profile/test.py | 5 + 27 files changed, 603 insertions(+), 289 deletions(-) diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 4dc91cd8937..8b249813f7c 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -265,7 +265,9 @@ namespace /// Calculates the path for storing a map of name of access entity to UUID for access entities of some type. std::filesystem::path getListFilePath(const String & directory_path, EntityType type) { - std::string_view file_name = EntityTypeInfo::get(type).list_filename; + String file_name = EntityTypeInfo::get(type).plural_raw_name; + boost::to_lower(file_name); + file_name += ".list"; return std::filesystem::path(directory_path).append(file_name); } diff --git a/src/Access/IAccessEntity.h b/src/Access/IAccessEntity.h index 39a5cefa7d7..68e14c99982 100644 --- a/src/Access/IAccessEntity.h +++ b/src/Access/IAccessEntity.h @@ -45,11 +45,13 @@ struct IAccessEntity struct TypeInfo { const char * const raw_name; + const char * const plural_raw_name; const String name; /// Uppercased with spaces instead of underscores, e.g. "SETTINGS PROFILE". const String alias; /// Alias of the keyword or empty string, e.g. "PROFILE". + const String plural_name; /// Uppercased with spaces plural name, e.g. "SETTINGS PROFILES". + const String plural_alias; /// Uppercased with spaces plural name alias, e.g. "PROFILES". const String name_for_output_with_entity_name; /// Lowercased with spaces instead of underscores, e.g. "settings profile". const char unique_char; /// Unique character for this type. E.g. 'P' for SETTINGS_PROFILE. - const String list_filename; /// Name of the file containing list of objects of this type, including the file extension ".list". const int not_found_error_code; static const TypeInfo & get(Type type_); @@ -69,6 +71,18 @@ struct IAccessEntity friend bool operator ==(const IAccessEntity & lhs, const IAccessEntity & rhs) { return lhs.equal(rhs); } friend bool operator !=(const IAccessEntity & lhs, const IAccessEntity & rhs) { return !(lhs == rhs); } + struct LessByName + { + bool operator()(const IAccessEntity & lhs, const IAccessEntity & rhs) const { return (lhs.getName() < rhs.getName()); } + bool operator()(const std::shared_ptr & lhs, const std::shared_ptr & rhs) const { return operator()(*lhs, *rhs); } + }; + + struct LessByTypeAndName + { + bool operator()(const IAccessEntity & lhs, const IAccessEntity & rhs) const { return (lhs.getType() < rhs.getType()) || ((lhs.getType() == rhs.getType()) && (lhs.getName() < rhs.getName())); } + bool operator()(const std::shared_ptr & lhs, const std::shared_ptr & rhs) const { return operator()(*lhs, *rhs); } + }; + protected: String name; @@ -87,44 +101,49 @@ using AccessEntityPtr = std::shared_ptr; inline const IAccessEntity::TypeInfo & IAccessEntity::TypeInfo::get(Type type_) { - static constexpr auto make_info = [](const char * raw_name_, char unique_char_, const char * list_filename_, int not_found_error_code_) + static constexpr auto make_info = [](const char * raw_name_, const char * plural_raw_name_, char unique_char_, int not_found_error_code_) { - String init_name = raw_name_; - boost::to_upper(init_name); - boost::replace_all(init_name, "_", " "); - String init_alias; - if (auto underscore_pos = init_name.find_first_of(" "); underscore_pos != String::npos) - init_alias = init_name.substr(underscore_pos + 1); - String init_name_for_output_with_entity_name = init_name; + String init_names[2] = {raw_name_, plural_raw_name_}; + String init_aliases[2]; + for (size_t i = 0; i != std::size(init_names); ++i) + { + String & init_name = init_names[i]; + String & init_alias = init_aliases[i]; + boost::to_upper(init_name); + boost::replace_all(init_name, "_", " "); + if (auto underscore_pos = init_name.find_first_of(" "); underscore_pos != String::npos) + init_alias = init_name.substr(underscore_pos + 1); + } + String init_name_for_output_with_entity_name = init_names[0]; boost::to_lower(init_name_for_output_with_entity_name); - return TypeInfo{raw_name_, std::move(init_name), std::move(init_alias), std::move(init_name_for_output_with_entity_name), unique_char_, list_filename_, not_found_error_code_}; + return TypeInfo{raw_name_, plural_raw_name_, std::move(init_names[0]), std::move(init_aliases[0]), std::move(init_names[1]), std::move(init_aliases[1]), std::move(init_name_for_output_with_entity_name), unique_char_, not_found_error_code_}; }; switch (type_) { case Type::USER: { - static const auto info = make_info("USER", 'U', "users.list", ErrorCodes::UNKNOWN_USER); + static const auto info = make_info("USER", "USERS", 'U', ErrorCodes::UNKNOWN_USER); return info; } case Type::ROLE: { - static const auto info = make_info("ROLE", 'R', "roles.list", ErrorCodes::UNKNOWN_ROLE); + static const auto info = make_info("ROLE", "ROLES", 'R', ErrorCodes::UNKNOWN_ROLE); return info; } case Type::SETTINGS_PROFILE: { - static const auto info = make_info("SETTINGS_PROFILE", 'S', "settings_profiles.list", ErrorCodes::THERE_IS_NO_PROFILE); + static const auto info = make_info("SETTINGS_PROFILE", "SETTINGS_PROFILES", 'S', ErrorCodes::THERE_IS_NO_PROFILE); return info; } case Type::ROW_POLICY: { - static const auto info = make_info("ROW_POLICY", 'P', "row_policies.list", ErrorCodes::UNKNOWN_ROW_POLICY); + static const auto info = make_info("ROW_POLICY", "ROW_POLICIES", 'P', ErrorCodes::UNKNOWN_ROW_POLICY); return info; } case Type::QUOTA: { - static const auto info = make_info("QUOTA", 'Q', "quotas.list", ErrorCodes::UNKNOWN_QUOTA); + static const auto info = make_info("QUOTA", "QUOTAS", 'Q', ErrorCodes::UNKNOWN_QUOTA); return info; } case Type::MAX: break; diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp index 8d54da790f8..379580fe58a 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp @@ -31,7 +31,8 @@ BlockIO InterpreterShowAccessEntitiesQuery::execute() String InterpreterShowAccessEntitiesQuery::getRewrittenQuery() const { - const auto & query = query_ptr->as(); + auto & query = query_ptr->as(); + query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); String origin; String expr = "*"; String filter, order; @@ -42,14 +43,18 @@ String InterpreterShowAccessEntitiesQuery::getRewrittenQuery() const { origin = "row_policies"; expr = "name"; - const String & table_name = query.table_name; - if (!table_name.empty()) + if (!query.short_name.empty()) + filter += String{filter.empty() ? "" : " AND "} + "short_name = " + quoteString(query.short_name); + if (query.database_and_table_name) { - String database = query.database; - if (database.empty()) - database = context.getCurrentDatabase(); - filter = "database = " + quoteString(database) + " AND table = " + quoteString(table_name); - expr = "short_name"; + const String & database = query.database_and_table_name->first; + const String & table_name = query.database_and_table_name->second; + if (!database.empty()) + filter += String{filter.empty() ? "" : " AND "} + "database = " + quoteString(database); + if (!table_name.empty()) + filter += String{filter.empty() ? "" : " AND "} + "table = " + quoteString(table_name); + if (!database.empty() && !table_name.empty()) + expr = "short_name"; } break; } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 5c4173d7aa3..55c5d961ad8 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -232,10 +233,8 @@ BlockIO InterpreterShowCreateAccessEntityQuery::execute() BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() { - auto & show_query = query_ptr->as(); - /// Build a create queries. - ASTs create_queries = getCreateQueries(show_query); + ASTs create_queries = getCreateQueries(); /// Build the result column. MutableColumnPtr column = ColumnString::create(); @@ -249,6 +248,7 @@ BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() /// Prepare description of the result column. std::stringstream desc_ss; + const auto & show_query = query_ptr->as(); formatAST(show_query, desc_ss, false, true); String desc = desc_ss.str(); String prefix = "SHOW "; @@ -259,52 +259,94 @@ BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() } -ASTs InterpreterShowCreateAccessEntityQuery::getCreateQueries(ASTShowCreateAccessEntityQuery & show_query) const +std::vector InterpreterShowCreateAccessEntityQuery::getEntities() const { + auto & show_query = query_ptr->as(); const auto & access_control = context.getAccessControlManager(); context.checkAccess(getRequiredAccess()); show_query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); + std::vector entities; - if (show_query.current_user) + if (show_query.all) { - auto user = context.getUser(); - if (!user) - return {}; - return {getCreateQueryImpl(*user, &access_control, false)}; + auto ids = access_control.findAll(show_query.type); + for (const auto & id : ids) + { + if (auto entity = access_control.tryRead(id)) + entities.push_back(entity); + } } - - if (show_query.current_quota) + else if (show_query.current_user) + { + if (auto user = context.getUser()) + entities.push_back(user); + } + else if (show_query.current_quota) { auto usage = context.getQuotaUsage(); - if (!usage) - return {}; - auto quota = access_control.read(usage->quota_id); - return {getCreateQueryImpl(*quota, &access_control, false)}; + if (usage) + entities.push_back(access_control.read(usage->quota_id)); } - - ASTs list; - - if (show_query.type == EntityType::ROW_POLICY) + else if (show_query.type == EntityType::ROW_POLICY) { - for (const String & name : show_query.row_policy_names->toStrings()) + auto ids = access_control.findAll(); + if (show_query.row_policy_names) { - RowPolicyPtr policy = access_control.read(name); - list.push_back(getCreateQueryImpl(*policy, &access_control, false)); + for (const String & name : show_query.row_policy_names->toStrings()) + entities.push_back(access_control.read(name)); + } + else + { + for (const auto & id : ids) + { + auto policy = access_control.tryRead(id); + if (!policy) + continue; + if (!show_query.short_name.empty() && (policy->getShortName() != show_query.short_name)) + continue; + if (show_query.database_and_table_name) + { + const String & database = show_query.database_and_table_name->first; + const String & table_name = show_query.database_and_table_name->second; + if (!database.empty() && (policy->getDatabase() != database)) + continue; + if (!table_name.empty() && (policy->getTableName() != table_name)) + continue; + } + entities.push_back(policy); + } } } else { for (const String & name : show_query.names) - { - auto entity = access_control.read(access_control.getID(show_query.type, name)); - list.push_back(getCreateQueryImpl(*entity, &access_control, false)); - } + entities.push_back(access_control.read(access_control.getID(show_query.type, name))); } + boost::range::sort(entities, IAccessEntity::LessByName{}); + return entities; +} + + +ASTs InterpreterShowCreateAccessEntityQuery::getCreateQueries() const +{ + auto entities = getEntities(); + + ASTs list; + const auto & access_control = context.getAccessControlManager(); + for (const auto & entity : entities) + list.push_back(getCreateQuery(*entity, access_control)); + return list; } +ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(const IAccessEntity & entity, const AccessControlManager & access_control) +{ + return getCreateQueryImpl(entity, &access_control, false); +} + + ASTPtr InterpreterShowCreateAccessEntityQuery::getAttachQuery(const IAccessEntity & entity) { return getCreateQueryImpl(entity, nullptr, true); diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h index 0d2978cff6c..12f427b5eb0 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h @@ -7,10 +7,11 @@ namespace DB { +class AccessControlManager; class Context; -class ASTShowCreateAccessEntityQuery; class AccessRightsElements; struct IAccessEntity; +using AccessEntityPtr = std::shared_ptr; /** Returns a single item containing a statement which could be used to create a specified role. @@ -25,11 +26,13 @@ public: bool ignoreQuota() const override { return ignore_quota; } bool ignoreLimits() const override { return ignore_quota; } + static ASTPtr getCreateQuery(const IAccessEntity & entity, const AccessControlManager & access_control); static ASTPtr getAttachQuery(const IAccessEntity & entity); private: BlockInputStreamPtr executeImpl(); - ASTs getCreateQueries(ASTShowCreateAccessEntityQuery & show_query) const; + std::vector getEntities() const; + ASTs getCreateQueries() const; AccessRightsElements getRequiredAccess() const; ASTPtr query_ptr; diff --git a/src/Interpreters/InterpreterShowGrantsQuery.cpp b/src/Interpreters/InterpreterShowGrantsQuery.cpp index c6e3ccce7c7..ebb0d871c8b 100644 --- a/src/Interpreters/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/InterpreterShowGrantsQuery.cpp @@ -10,6 +10,9 @@ #include #include #include +#include +#include +#include namespace DB @@ -121,10 +124,8 @@ BlockIO InterpreterShowGrantsQuery::execute() BlockInputStreamPtr InterpreterShowGrantsQuery::executeImpl() { - const auto & show_query = query_ptr->as(); - /// Build a create query. - ASTs grant_queries = getGrantQueries(show_query); + ASTs grant_queries = getGrantQueries(); /// Build the result column. MutableColumnPtr column = ColumnString::create(); @@ -138,6 +139,7 @@ BlockInputStreamPtr InterpreterShowGrantsQuery::executeImpl() /// Prepare description of the result column. std::stringstream desc_ss; + const auto & show_query = query_ptr->as(); formatAST(show_query, desc_ss, false, true); String desc = desc_ss.str(); String prefix = "SHOW "; @@ -148,21 +150,41 @@ BlockInputStreamPtr InterpreterShowGrantsQuery::executeImpl() } -ASTs InterpreterShowGrantsQuery::getGrantQueries(const ASTShowGrantsQuery & show_query) const +std::vector InterpreterShowGrantsQuery::getEntities() const { + const auto & show_query = query_ptr->as(); const auto & access_control = context.getAccessControlManager(); + auto ids = RolesOrUsersSet{*show_query.for_roles, access_control, context.getUserID()}.getMatchingIDs(access_control); - AccessEntityPtr user_or_role; - if (show_query.current_user) - user_or_role = context.getUser(); - else + std::vector entities; + for (const auto & id : ids) { - user_or_role = access_control.tryRead(show_query.name); - if (!user_or_role) - user_or_role = access_control.read(show_query.name); + auto entity = access_control.tryRead(id); + if (entity) + entities.push_back(entity); } - return getGrantQueriesImpl(*user_or_role, &access_control); + boost::range::sort(entities, IAccessEntity::LessByTypeAndName{}); + return entities; +} + + +ASTs InterpreterShowGrantsQuery::getGrantQueries() const +{ + auto entities = getEntities(); + const auto & access_control = context.getAccessControlManager(); + + ASTs grant_queries; + for (const auto & entity : entities) + boost::range::push_back(grant_queries, getGrantQueries(*entity, access_control)); + + return grant_queries; +} + + +ASTs InterpreterShowGrantsQuery::getGrantQueries(const IAccessEntity & user_or_role, const AccessControlManager & access_control) +{ + return getGrantQueriesImpl(user_or_role, &access_control, false); } diff --git a/src/Interpreters/InterpreterShowGrantsQuery.h b/src/Interpreters/InterpreterShowGrantsQuery.h index b400130a292..af6c3065192 100644 --- a/src/Interpreters/InterpreterShowGrantsQuery.h +++ b/src/Interpreters/InterpreterShowGrantsQuery.h @@ -7,8 +7,10 @@ namespace DB { +class AccessControlManager; class ASTShowGrantsQuery; struct IAccessEntity; +using AccessEntityPtr = std::shared_ptr; class InterpreterShowGrantsQuery : public IInterpreter @@ -18,11 +20,13 @@ public: BlockIO execute() override; + static ASTs getGrantQueries(const IAccessEntity & user_or_role, const AccessControlManager & access_control); static ASTs getAttachGrantQueries(const IAccessEntity & user_or_role); private: BlockInputStreamPtr executeImpl(); - ASTs getGrantQueries(const ASTShowGrantsQuery & show_query) const; + ASTs getGrantQueries() const; + std::vector getEntities() const; ASTPtr query_ptr; Context & context; diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.cpp b/src/Parsers/ASTShowAccessEntitiesQuery.cpp index cb1ccff5273..e87baebba33 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.cpp +++ b/src/Parsers/ASTShowAccessEntitiesQuery.cpp @@ -4,49 +4,51 @@ namespace DB { -namespace ErrorCodes -{ - extern const int NOT_IMPLEMENTED; -} +using EntityTypeInfo = IAccessEntity::TypeInfo; -const char * ASTShowAccessEntitiesQuery::getKeyword() const +String ASTShowAccessEntitiesQuery::getKeyword() const { - switch (type) - { - case EntityType::ROW_POLICY: - return "SHOW ROW POLICIES"; - case EntityType::QUOTA: - return current_quota ? "SHOW CURRENT QUOTA" : "SHOW QUOTAS"; - case EntityType::SETTINGS_PROFILE: - return "SHOW SETTINGS PROFILES"; - case EntityType::USER: - return "SHOW USERS"; - case EntityType::ROLE: - return current_roles ? "SHOW CURRENT ROLES" : (enabled_roles ? "SHOW ENABLED ROLES" : "SHOW ROLES"); - case EntityType::MAX: - break; - } - throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); + if (current_quota) + return "CURRENT QUOTA"; + if (current_roles) + return "CURRENT ROLES"; + if (enabled_roles) + return "ENABLED ROLES"; + return EntityTypeInfo::get(type).plural_name; } String ASTShowAccessEntitiesQuery::getID(char) const { - return String(getKeyword()) + " query"; + return "SHOW " + String(getKeyword()) + " query"; } void ASTShowAccessEntitiesQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { - const char * keyword = getKeyword(); - settings.ostr << (settings.hilite ? hilite_keyword : "") << keyword << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << getKeyword() << (settings.hilite ? hilite_none : ""); - if ((type == EntityType::ROW_POLICY) && !table_name.empty()) + if (!short_name.empty()) + settings.ostr << " " << backQuoteIfNeed(short_name); + + if (database_and_table_name) { + const String & database = database_and_table_name->first; + const String & table_name = database_and_table_name->second; settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); - if (!database.empty()) - settings.ostr << backQuoteIfNeed(database) << "."; - settings.ostr << backQuoteIfNeed(table_name); + settings.ostr << (database.empty() ? "" : backQuoteIfNeed(database) + "."); + settings.ostr << (table_name.empty() ? "*" : backQuoteIfNeed(table_name)); + } +} + + +void ASTShowAccessEntitiesQuery::replaceEmptyDatabaseWithCurrent(const String & current_database) +{ + if (database_and_table_name) + { + String & database = database_and_table_name->first; + if (database.empty()) + database = current_database; } } diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.h b/src/Parsers/ASTShowAccessEntitiesQuery.h index 3bf16ad6abd..7ccd76bfe5e 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.h +++ b/src/Parsers/ASTShowAccessEntitiesQuery.h @@ -7,32 +7,37 @@ namespace DB { -/// SHOW [ROW] POLICIES [ON [database.]table] -/// SHOW QUOTAS -/// SHOW [CURRENT] QUOTA -/// SHOW [SETTINGS] PROFILES /// SHOW USERS /// SHOW [CURRENT|ENABLED] ROLES +/// SHOW [SETTINGS] PROFILES +/// SHOW [ROW] POLICIES [name | ON [database.]table] +/// SHOW QUOTAS +/// SHOW [CURRENT] QUOTA class ASTShowAccessEntitiesQuery : public ASTQueryWithOutput { public: using EntityType = IAccessEntity::Type; EntityType type; - String database; - String table_name; + + bool all = false; bool current_quota = false; bool current_roles = false; bool enabled_roles = false; + String short_name; + std::optional> database_and_table_name; + String getID(char) const override; ASTPtr clone() const override { return std::make_shared(*this); } + void replaceEmptyDatabaseWithCurrent(const String & current_database); + protected: void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: - const char * getKeyword() const; + String getKeyword() const; }; } diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp index cbd31d0d53c..bc309ab5c44 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp @@ -7,6 +7,7 @@ namespace DB { namespace { + using EntityType = IAccessEntity::Type; using EntityTypeInfo = IAccessEntity::TypeInfo; void formatNames(const Strings & names, const IAST::FormatSettings & settings) @@ -22,9 +23,18 @@ namespace } +String ASTShowCreateAccessEntityQuery::getKeyword() const +{ + size_t total_count = (names.size()) + (row_policy_names ? row_policy_names->size() : 0) + current_user + current_quota; + bool multiple = (total_count != 1) || all || !short_name.empty() || database_and_table_name; + const auto & type_info = EntityTypeInfo::get(type); + return multiple ? type_info.plural_name : type_info.name; +} + + String ASTShowCreateAccessEntityQuery::getID(char) const { - return String("SHOW CREATE ") + toString(type) + " query"; + return String("SHOW CREATE ") + getKeyword() + " query"; } @@ -36,20 +46,28 @@ ASTPtr ASTShowCreateAccessEntityQuery::clone() const void ASTShowCreateAccessEntityQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") - << "SHOW CREATE " << EntityTypeInfo::get(type).name - << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CREATE " << getKeyword() << (settings.hilite ? hilite_none : ""); - if (current_user || current_quota) - { - } - else if (type == EntityType::ROW_POLICY) + if (!names.empty()) + formatNames(names, settings); + + if (row_policy_names) { settings.ostr << " "; row_policy_names->format(settings); } - else - formatNames(names, settings); + + if (!short_name.empty()) + settings.ostr << " " << backQuoteIfNeed(short_name); + + if (database_and_table_name) + { + const String & database = database_and_table_name->first; + const String & table_name = database_and_table_name->second; + settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); + settings.ostr << (database.empty() ? "" : backQuoteIfNeed(database) + "."); + settings.ostr << (table_name.empty() ? "*" : backQuoteIfNeed(table_name)); + } } @@ -57,6 +75,13 @@ void ASTShowCreateAccessEntityQuery::replaceEmptyDatabaseWithCurrent(const Strin { if (row_policy_names) row_policy_names->replaceEmptyDatabaseWithCurrent(current_database); + + if (database_and_table_name) + { + String & database = database_and_table_name->first; + if (database.empty()) + database = current_database; + } } } diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.h b/src/Parsers/ASTShowCreateAccessEntityQuery.h index 5fd16d622f7..f112e9211fe 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.h +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.h @@ -8,11 +8,16 @@ namespace DB { class ASTRowPolicyNames; -/** SHOW CREATE QUOTA [name] - * SHOW CREATE [ROW] POLICY name ON [database.]table - * SHOW CREATE USER [name | CURRENT_USER] +/** SHOW CREATE USER [name | CURRENT_USER] + * SHOW CREATE USERS [name [, name2 ...] * SHOW CREATE ROLE name + * SHOW CREATE ROLES [name [, name2 ...]] * SHOW CREATE [SETTINGS] PROFILE name + * SHOW CREATE [SETTINGS] PROFILES [name [, name2 ...]] + * SHOW CREATE [ROW] POLICY name ON [database.]table + * SHOW CREATE [ROW] POLICIES [name ON [database.]table [, name2 ON database2.table2 ...] | name | ON database.table] + * SHOW CREATE QUOTA [name] + * SHOW CREATE QUOTAS [name [, name2 ...]] */ class ASTShowCreateAccessEntityQuery : public ASTQueryWithOutput { @@ -21,9 +26,14 @@ public: EntityType type; Strings names; + std::shared_ptr row_policy_names; + bool current_quota = false; bool current_user = false; - std::shared_ptr row_policy_names; + bool all = false; + + String short_name; + std::optional> database_and_table_name; String getID(char) const override; ASTPtr clone() const override; @@ -31,6 +41,7 @@ public: void replaceEmptyDatabaseWithCurrent(const String & current_database); protected: + String getKeyword() const; void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; diff --git a/src/Parsers/ASTShowGrantsQuery.cpp b/src/Parsers/ASTShowGrantsQuery.cpp index b3cc0cbd386..26ae506d7d4 100644 --- a/src/Parsers/ASTShowGrantsQuery.cpp +++ b/src/Parsers/ASTShowGrantsQuery.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -21,8 +22,15 @@ void ASTShowGrantsQuery::formatQueryImpl(const FormatSettings & settings, Format settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW GRANTS" << (settings.hilite ? hilite_none : ""); - if (!current_user) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " FOR " << (settings.hilite ? hilite_none : "") - << backQuoteIfNeed(name); + if (for_roles->current_user && !for_roles->all && for_roles->names.empty() && for_roles->except_names.empty() + && !for_roles->except_current_user) + { + } + else + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " FOR " + << (settings.hilite ? hilite_none : ""); + for_roles->format(settings); + } } } diff --git a/src/Parsers/ASTShowGrantsQuery.h b/src/Parsers/ASTShowGrantsQuery.h index 1afb5e5ac0d..04764fe3502 100644 --- a/src/Parsers/ASTShowGrantsQuery.h +++ b/src/Parsers/ASTShowGrantsQuery.h @@ -5,13 +5,14 @@ namespace DB { +class ASTRolesOrUsersSet; + /** SHOW GRANTS [FOR user_name] */ class ASTShowGrantsQuery : public ASTQueryWithOutput { public: - String name; - bool current_user = false; + std::shared_ptr for_roles; String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ParserDropAccessEntityQuery.cpp b/src/Parsers/ParserDropAccessEntityQuery.cpp index d5eac710631..b1f3ce5f93d 100644 --- a/src/Parsers/ParserDropAccessEntityQuery.cpp +++ b/src/Parsers/ParserDropAccessEntityQuery.cpp @@ -15,6 +15,23 @@ namespace using EntityType = IAccessEntity::Type; using EntityTypeInfo = IAccessEntity::TypeInfo; + + bool parseEntityType(IParserBase::Pos & pos, Expected & expected, EntityType & type) + { + for (auto i : ext::range(EntityType::MAX)) + { + const auto & type_info = EntityTypeInfo::get(i); + if (ParserKeyword{type_info.name.c_str()}.ignore(pos, expected) + || (!type_info.alias.empty() && ParserKeyword{type_info.alias.c_str()}.ignore(pos, expected))) + { + type = i; + return true; + } + } + return false; + } + + bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster) { return IParserBase::wrapParseImpl(pos, [&] @@ -30,17 +47,8 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & if (!ParserKeyword{"DROP"}.ignore(pos, expected)) return false; - std::optional type; - for (auto type_i : ext::range(EntityType::MAX)) - { - const auto & type_info = EntityTypeInfo::get(type_i); - if (ParserKeyword{type_info.name.c_str()}.ignore(pos, expected) - || (!type_info.alias.empty() && ParserKeyword{type_info.alias.c_str()}.ignore(pos, expected))) - { - type = type_i; - } - } - if (!type) + EntityType type; + if (!parseEntityType(pos, expected, type)) return false; bool if_exists = false; @@ -78,7 +86,7 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & auto query = std::make_shared(); node = query; - query->type = *type; + query->type = type; query->if_exists = if_exists; query->cluster = std::move(cluster); query->names = std::move(names); diff --git a/src/Parsers/ParserGrantQuery.cpp b/src/Parsers/ParserGrantQuery.cpp index 5eb5353f2ee..62efd5314ac 100644 --- a/src/Parsers/ParserGrantQuery.cpp +++ b/src/Parsers/ParserGrantQuery.cpp @@ -1,13 +1,12 @@ #include #include -#include #include #include #include -#include #include #include #include +#include #include @@ -85,72 +84,6 @@ namespace }); } - - bool parseDatabaseAndTableNameOrMaybeAsterisks( - IParser::Pos & pos, Expected & expected, String & database_name, bool & any_database, String & table_name, bool & any_table) - { - return IParserBase::wrapParseImpl(pos, [&] - { - ASTPtr ast[2]; - if (ParserToken{TokenType::Asterisk}.ignore(pos, expected)) - { - if (ParserToken{TokenType::Dot}.ignore(pos, expected)) - { - if (!ParserToken{TokenType::Asterisk}.ignore(pos, expected)) - return false; - - /// *.* (any table in any database) - any_database = true; - database_name.clear(); - any_table = true; - table_name.clear(); - return true; - } - - /// * (any table in the current database) - any_database = false; - database_name.clear(); - any_table = true; - table_name.clear(); - return true; - } - - if (!ParserIdentifier().parse(pos, ast[0], expected)) - return false; - - if (ParserToken{TokenType::Dot}.ignore(pos, expected)) - { - if (ParserToken{TokenType::Asterisk}.ignore(pos, expected)) - { - /// .* - any_database = false; - database_name = getIdentifierName(ast[0]); - any_table = true; - table_name.clear(); - return true; - } - - if (!ParserIdentifier().parse(pos, ast[1], expected)) - return false; - - /// . - any_database = false; - database_name = getIdentifierName(ast[0]); - any_table = false; - table_name = getIdentifierName(ast[1]); - return true; - } - - /// - the current database, specified table - any_database = false; - database_name.clear(); - any_table = false; - table_name = getIdentifierName(ast[0]); - return true; - }); - } - - bool parseAccessTypesWithColumns(IParser::Pos & pos, Expected & expected, std::vector> & access_and_columns) { @@ -193,7 +126,7 @@ namespace String database_name, table_name; bool any_database = false, any_table = false; - if (!parseDatabaseAndTableNameOrMaybeAsterisks(pos, expected, database_name, any_database, table_name, any_table)) + if (!parseDatabaseAndTableNameOrAsterisks(pos, expected, database_name, any_database, table_name, any_table)) return false; for (auto & [access_flags, columns] : access_and_columns) diff --git a/src/Parsers/ParserShowAccessEntitiesQuery.cpp b/src/Parsers/ParserShowAccessEntitiesQuery.cpp index c50bd5b402c..f070641cbd2 100644 --- a/src/Parsers/ParserShowAccessEntitiesQuery.cpp +++ b/src/Parsers/ParserShowAccessEntitiesQuery.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include namespace DB @@ -9,14 +11,29 @@ namespace DB namespace { using EntityType = IAccessEntity::Type; + using EntityTypeInfo = IAccessEntity::TypeInfo; - bool parseONDatabaseAndTableName(IParserBase::Pos & pos, Expected & expected, String & database, String & table_name) + bool parseEntityType(IParserBase::Pos & pos, Expected & expected, EntityType & type) + { + for (auto i : ext::range(EntityType::MAX)) + { + const auto & type_info = EntityTypeInfo::get(i); + if (ParserKeyword{type_info.plural_name.c_str()}.ignore(pos, expected) + || (!type_info.plural_alias.empty() && ParserKeyword{type_info.plural_alias.c_str()}.ignore(pos, expected))) + { + type = i; + return true; + } + } + return false; + } + + bool parseOnDBAndTableName(IParserBase::Pos & pos, Expected & expected, String & database, bool & any_database, String & table, bool & any_table) { return IParserBase::wrapParseImpl(pos, [&] { - database.clear(); - table_name.clear(); - return ParserKeyword{"ON"}.ignore(pos, expected) && parseDatabaseAndTableName(pos, expected, database, table_name); + return ParserKeyword{"ON"}.ignore(pos, expected) + && parseDatabaseAndTableNameOrAsterisks(pos, expected, database, any_database, table, any_table); }); } } @@ -27,18 +44,15 @@ bool ParserShowAccessEntitiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected if (!ParserKeyword{"SHOW"}.ignore(pos, expected)) return false; - std::optional type; + EntityType type; + bool all = false; bool current_quota = false; bool current_roles = false; bool enabled_roles = false; - if (ParserKeyword{"USERS"}.ignore(pos, expected)) + if (parseEntityType(pos, expected, type)) { - type = EntityType::USER; - } - else if (ParserKeyword{"ROLES"}.ignore(pos, expected)) - { - type = EntityType::ROLE; + all = true; } else if (ParserKeyword{"CURRENT ROLES"}.ignore(pos, expected)) { @@ -50,39 +64,44 @@ bool ParserShowAccessEntitiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected type = EntityType::ROLE; enabled_roles = true; } - else if (ParserKeyword{"POLICIES"}.ignore(pos, expected) || ParserKeyword{"ROW POLICIES"}.ignore(pos, expected)) - { - type = EntityType::ROW_POLICY; - } - else if (ParserKeyword{"QUOTAS"}.ignore(pos, expected)) - { - type = EntityType::QUOTA; - } - else if (ParserKeyword{"QUOTA"}.ignore(pos, expected) || ParserKeyword{"CURRENT QUOTA"}.ignore(pos, expected)) + else if (ParserKeyword{"CURRENT QUOTA"}.ignore(pos, expected) || ParserKeyword{"QUOTA"}.ignore(pos, expected)) { type = EntityType::QUOTA; current_quota = true; } - else if (ParserKeyword{"PROFILES"}.ignore(pos, expected) || ParserKeyword{"SETTINGS PROFILES"}.ignore(pos, expected)) - { - type = EntityType::SETTINGS_PROFILE; - } else return false; - String database, table_name; + String short_name; + std::optional> database_and_table_name; if (type == EntityType::ROW_POLICY) - parseONDatabaseAndTableName(pos, expected, database, table_name); + { + String database, table_name; + bool any_database, any_table; + if (parseOnDBAndTableName(pos, expected, database, any_database, table_name, any_table)) + { + if (any_database) + all = true; + else + database_and_table_name.emplace(database, table_name); + } + else if (parseIdentifierOrStringLiteral(pos, expected, short_name)) + { + } + else + all = true; + } auto query = std::make_shared(); node = query; - query->type = *type; + query->type = type; + query->all = all; query->current_quota = current_quota; query->current_roles = current_roles; query->enabled_roles = enabled_roles; - query->database = std::move(database); - query->table_name = std::move(table_name); + query->short_name = std::move(short_name); + query->database_and_table_name = std::move(database_and_table_name); return true; } diff --git a/src/Parsers/ParserShowAccessEntitiesQuery.h b/src/Parsers/ParserShowAccessEntitiesQuery.h index bb8b37f40e8..fcee3b09868 100644 --- a/src/Parsers/ParserShowAccessEntitiesQuery.h +++ b/src/Parsers/ParserShowAccessEntitiesQuery.h @@ -6,10 +6,12 @@ namespace DB { /** Parses queries like - * SHOW [ROW] POLICIES [ON [database.]table] - SHOW QUOTAS - SHOW [CURRENT] QUOTA - SHOW [SETTINGS] PROFILES + * SHOW USERS + * SHOW [CURRENT|ENABLED] ROLES + * SHOW [SETTINGS] PROFILES + * SHOW [ROW] POLICIES [name | ON [database.]table] + * SHOW QUOTAS + * SHOW [CURRENT] QUOTA */ class ParserShowAccessEntitiesQuery : public IParserBase { diff --git a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp index 465a6c380b1..55d00572e48 100644 --- a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp @@ -5,14 +5,62 @@ #include #include #include +#include #include #include namespace DB { -using EntityType = IAccessEntity::Type; -using EntityTypeInfo = IAccessEntity::TypeInfo; +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + + +namespace +{ + using EntityType = IAccessEntity::Type; + using EntityTypeInfo = IAccessEntity::TypeInfo; + + bool parseEntityType(IParserBase::Pos & pos, Expected & expected, EntityType & type, bool & plural) + { + for (auto i : ext::range(EntityType::MAX)) + { + const auto & type_info = EntityTypeInfo::get(i); + if (ParserKeyword{type_info.name.c_str()}.ignore(pos, expected) + || (!type_info.alias.empty() && ParserKeyword{type_info.alias.c_str()}.ignore(pos, expected))) + { + type = i; + plural = false; + return true; + } + } + + for (auto i : ext::range(EntityType::MAX)) + { + const auto & type_info = EntityTypeInfo::get(i); + if (ParserKeyword{type_info.plural_name.c_str()}.ignore(pos, expected) + || (!type_info.plural_alias.empty() && ParserKeyword{type_info.plural_alias.c_str()}.ignore(pos, expected))) + { + type = i; + plural = true; + return true; + } + } + + return false; + } + + bool parseOnDBAndTableName(IParserBase::Pos & pos, Expected & expected, String & database, bool & any_database, String & table, bool & any_table) + { + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{"ON"}.ignore(pos, expected) + && parseDatabaseAndTableNameOrAsterisks(pos, expected, database, any_database, table, any_table); + }); + } +} bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) @@ -20,65 +68,105 @@ bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expe if (!ParserKeyword{"SHOW CREATE"}.ignore(pos, expected)) return false; - std::optional type; - for (auto type_i : ext::range(EntityType::MAX)) - { - const auto & type_info = EntityTypeInfo::get(type_i); - if (ParserKeyword{type_info.name.c_str()}.ignore(pos, expected) - || (!type_info.alias.empty() && ParserKeyword{type_info.alias.c_str()}.ignore(pos, expected))) - { - type = type_i; - } - } - if (!type) + EntityType type; + bool plural; + if (!parseEntityType(pos, expected, type, plural)) return false; Strings names; + std::shared_ptr row_policy_names; + bool all = false; bool current_quota = false; bool current_user = false; - std::shared_ptr row_policy_names; + String short_name; + std::optional> database_and_table_name; - if (type == EntityType::USER) + switch (type) { - if (parseCurrentUserTag(pos, expected)) - current_user = true; - else if (!parseUserNames(pos, expected, names)) - return false; - } - else if (type == EntityType::ROLE) - { - if (!parseRoleNames(pos, expected, names)) - return false; - } - else if (type == EntityType::ROW_POLICY) - { - ASTPtr ast; - if (!ParserRowPolicyNames{}.parse(pos, ast, expected)) - return false; - row_policy_names = typeid_cast>(ast); - } - else if (type == EntityType::QUOTA) - { - if (!parseIdentifiersOrStringLiterals(pos, expected, names)) + case EntityType::USER: { - /// SHOW CREATE QUOTA - current_quota = true; + if (parseCurrentUserTag(pos, expected)) + current_user = true; + else if (parseUserNames(pos, expected, names)) + { + } + else if (plural) + all = true; + else + current_user = true; + break; } - } - else if (type == EntityType::SETTINGS_PROFILE) - { - if (!parseIdentifiersOrStringLiterals(pos, expected, names)) - return false; + case EntityType::ROLE: + { + if (parseRoleNames(pos, expected, names)) + { + } + else if (plural) + all = true; + else + return false; + break; + } + case EntityType::ROW_POLICY: + { + ASTPtr ast; + String database, table_name; + bool any_database, any_table; + if (ParserRowPolicyNames{}.parse(pos, ast, expected)) + row_policy_names = typeid_cast>(ast); + else if (parseOnDBAndTableName(pos, expected, database, any_database, table_name, any_table)) + { + if (any_database) + all = true; + else + database_and_table_name.emplace(database, table_name); + } + else if (parseIdentifierOrStringLiteral(pos, expected, short_name)) + { + } + else if (plural) + all = true; + else + return false; + break; + } + case EntityType::SETTINGS_PROFILE: + { + if (parseIdentifiersOrStringLiterals(pos, expected, names)) + { + } + else if (plural) + all = true; + else + return false; + break; + } + case EntityType::QUOTA: + { + if (parseIdentifiersOrStringLiterals(pos, expected, names)) + { + } + else if (plural) + all = true; + else + current_quota = true; + break; + } + case EntityType::MAX: + throw Exception("Type " + toString(type) + " is not implemented in SHOW CREATE query", ErrorCodes::NOT_IMPLEMENTED); } auto query = std::make_shared(); node = query; - query->type = *type; + query->type = type; query->names = std::move(names); query->current_quota = current_quota; query->current_user = current_user; query->row_policy_names = std::move(row_policy_names); + query->all = all; + query->short_name = std::move(short_name); + query->database_and_table_name = std::move(database_and_table_name); return true; } diff --git a/src/Parsers/ParserShowCreateAccessEntityQuery.h b/src/Parsers/ParserShowCreateAccessEntityQuery.h index 025949d7fca..1a3148727e0 100644 --- a/src/Parsers/ParserShowCreateAccessEntityQuery.h +++ b/src/Parsers/ParserShowCreateAccessEntityQuery.h @@ -6,7 +6,16 @@ namespace DB { /** Parses queries like + * SHOW CREATE USER [name | CURRENT_USER] + * SHOW CREATE USERS [name [, name2 ...] + * SHOW CREATE ROLE name + * SHOW CREATE ROLES [name [, name2 ...]] + * SHOW CREATE [SETTINGS] PROFILE name + * SHOW CREATE [SETTINGS] PROFILES [name [, name2 ...]] + * SHOW CREATE [ROW] POLICY name ON [database.]table + * SHOW CREATE [ROW] POLICIES [name ON [database.]table [, name2 ON database2.table2 ...] | name | ON database.table] * SHOW CREATE QUOTA [name] + * SHOW CREATE QUOTAS [name [, name2 ...]] */ class ParserShowCreateAccessEntityQuery : public IParserBase { diff --git a/src/Parsers/ParserShowGrantsQuery.cpp b/src/Parsers/ParserShowGrantsQuery.cpp index 993346d2eeb..d25527754be 100644 --- a/src/Parsers/ParserShowGrantsQuery.cpp +++ b/src/Parsers/ParserShowGrantsQuery.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -11,25 +13,28 @@ bool ParserShowGrantsQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!ParserKeyword{"SHOW GRANTS"}.ignore(pos, expected)) return false; - String name; - bool current_user = false; + std::shared_ptr for_roles; if (ParserKeyword{"FOR"}.ignore(pos, expected)) { - if (parseCurrentUserTag(pos, expected)) - current_user = true; - else if (!parseUserName(pos, expected, name)) + ASTPtr for_roles_ast; + ParserRolesOrUsersSet for_roles_p; + for_roles_p.allowUserNames().allowRoleNames().allowAll().allowCurrentUser(); + if (!for_roles_p.parse(pos, for_roles_ast, expected)) return false; + + for_roles = typeid_cast>(for_roles_ast); } else - current_user = true; + { + for_roles = std::make_shared(); + for_roles->current_user = true; + } auto query = std::make_shared(); + query->for_roles = std::move(for_roles); node = query; - query->name = name; - query->current_user = current_user; - return true; } } diff --git a/src/Parsers/parseDatabaseAndTableName.cpp b/src/Parsers/parseDatabaseAndTableName.cpp index 018fee10731..13429df5b4d 100644 --- a/src/Parsers/parseDatabaseAndTableName.cpp +++ b/src/Parsers/parseDatabaseAndTableName.cpp @@ -41,4 +41,74 @@ bool parseDatabaseAndTableName(IParser::Pos & pos, Expected & expected, String & return true; } + +bool parseDatabaseAndTableNameOrAsterisks(IParser::Pos & pos, Expected & expected, String & database, bool & any_database, String & table, bool & any_table) +{ + return IParserBase::wrapParseImpl(pos, [&] + { + if (ParserToken{TokenType::Asterisk}.ignore(pos, expected)) + { + auto pos_before_dot = pos; + if (ParserToken{TokenType::Dot}.ignore(pos, expected) + && ParserToken{TokenType::Asterisk}.ignore(pos, expected)) + { + /// *.* + any_database = true; + database.clear(); + any_table = true; + table.clear(); + return true; + } + + /// * + pos = pos_before_dot; + any_database = false; + database.clear(); + any_table = true; + table.clear(); + return true; + } + + ASTPtr ast; + ParserIdentifier identifier_parser; + if (identifier_parser.parse(pos, ast, expected)) + { + String first_identifier = getIdentifierName(ast); + auto pos_before_dot = pos; + + if (ParserToken{TokenType::Dot}.ignore(pos, expected)) + { + if (ParserToken{TokenType::Asterisk}.ignore(pos, expected)) + { + /// db.* + any_database = false; + database = std::move(first_identifier); + any_table = true; + table.clear(); + return true; + } + else if (identifier_parser.parse(pos, ast, expected)) + { + /// db.table + any_database = false; + database = std::move(first_identifier); + any_table = false; + table = getIdentifierName(ast); + return true; + } + } + + /// table + pos = pos_before_dot; + any_database = false; + database.clear(); + any_table = false; + table = std::move(first_identifier); + return true; + } + + return false; + }); +} + } diff --git a/src/Parsers/parseDatabaseAndTableName.h b/src/Parsers/parseDatabaseAndTableName.h index aae78a2da20..e4699c8ad91 100644 --- a/src/Parsers/parseDatabaseAndTableName.h +++ b/src/Parsers/parseDatabaseAndTableName.h @@ -4,7 +4,10 @@ namespace DB { -/// Parses [db].name +/// Parses [db.]name bool parseDatabaseAndTableName(IParser::Pos & pos, Expected & expected, String & database_str, String & table_str); +/// Parses [db.]name or [db.]* or [*.]* +bool parseDatabaseAndTableNameOrAsterisks(IParser::Pos & pos, Expected & expected, String & database, bool & any_database, String & table, bool & any_table); + } diff --git a/tests/integration/test_create_user_and_login/test.py b/tests/integration/test_create_user_and_login/test.py index 32bf0af6bb6..8cfc1c2c9dd 100644 --- a/tests/integration/test_create_user_and_login/test.py +++ b/tests/integration/test_create_user_and_login/test.py @@ -69,8 +69,16 @@ def test_introspection(): instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION') assert instance.query("SHOW USERS") == TSV([ "A", "B", "default" ]) + assert instance.query("SHOW CREATE USERS A") == TSV([ "CREATE USER A" ]) + assert instance.query("SHOW CREATE USERS B") == TSV([ "CREATE USER B" ]) + assert instance.query("SHOW CREATE USERS A,B") == TSV([ "CREATE USER A", "CREATE USER B" ]) + assert instance.query("SHOW CREATE USERS") == TSV([ "CREATE USER A", "CREATE USER B", "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" ]) + assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A" ]) assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) + assert instance.query("SHOW GRANTS FOR A,B") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) + assert instance.query("SHOW GRANTS FOR B,A") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) + assert instance.query("SHOW GRANTS FOR ALL") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT ALL ON *.* TO default WITH GRANT OPTION" ]) assert instance.query("SHOW GRANTS", user='A') == TSV([ "GRANT SELECT ON test.table TO A" ]) assert instance.query("SHOW GRANTS", user='B') == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py index ab8077030e6..b9a6d80e54e 100644 --- a/tests/integration/test_quota/test.py +++ b/tests/integration/test_quota/test.py @@ -167,7 +167,6 @@ def test_add_remove_quota(): # Add quota. copy_quota_xml('two_quotas.xml') - print system_quotas() assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"], ["myQuota2", "4590510c-4d13-bf21-ec8a-c2187b092e73", "users.xml", "['client_key','user_name']", "[3600,2629746]", 0, "[]", "[]"]] assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], @@ -208,6 +207,7 @@ def test_reload_users_xml_by_timer(): def test_dcl_introspection(): assert instance.query("SHOW QUOTAS") == "myQuota\n" assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" + assert instance.query("SHOW CREATE QUOTAS") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t0\\t1000\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t1000\\t0\\t\\\\N\\t.*\\t\\\\N\n", instance.query("SHOW QUOTA")) @@ -228,6 +228,8 @@ def test_dcl_introspection(): assert instance.query("SHOW QUOTAS") == "myQuota\nmyQuota2\n" assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" assert instance.query("SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" + assert instance.query("SHOW CREATE QUOTAS") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n"\ + "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", instance.query("SHOW QUOTA")) diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py index 92e9f00d326..6bae3baafee 100644 --- a/tests/integration/test_role/test.py +++ b/tests/integration/test_role/test.py @@ -109,6 +109,11 @@ def test_introspection(): instance.query('REVOKE SELECT(x) ON test.table FROM R2') assert instance.query("SHOW ROLES") == TSV([ "R1", "R2" ]) + assert instance.query("SHOW CREATE ROLE R1") == TSV([ "CREATE ROLE R1" ]) + assert instance.query("SHOW CREATE ROLE R2") == TSV([ "CREATE ROLE R2" ]) + assert instance.query("SHOW CREATE ROLES R1, R2") == TSV([ "CREATE ROLE R1", "CREATE ROLE R2" ]) + assert instance.query("SHOW CREATE ROLES") == TSV([ "CREATE ROLE R1", "CREATE ROLE R2" ]) + assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT R1 TO A" ]) assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION" ]) assert instance.query("SHOW GRANTS FOR R1") == "" diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index 71496c6dbf2..d5fbae654a8 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -225,13 +225,21 @@ def test_introspection(): def test_dcl_introspection(): assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "another ON mydb.local", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3", "default ON mydb.local"]) - assert node.query("SHOW POLICIES ON mydb.filtered_table1") == TSV(["another", "default"]) - assert node.query("SHOW POLICIES ON mydb.local") == TSV(["another", "default"]) + + assert node.query("SHOW POLICIES ON mydb.filtered_table1") == TSV([ "another", "default" ]) + assert node.query("SHOW POLICIES ON mydb.local") == TSV([ "another", "default" ]) + assert node.query("SHOW POLICIES ON mydb.*") == TSV([ "another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "another ON mydb.local", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3", "default ON mydb.local" ]) + assert node.query("SHOW POLICIES default") == TSV([ "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3", "default ON mydb.local" ]) assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" assert node.query("SHOW CREATE POLICY default ON mydb.local") == "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" + + assert node.query("SHOW CREATE POLICY default") == TSV([ "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default" ]) + assert node.query("SHOW CREATE POLICIES ON mydb.filtered_table1") == TSV([ "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default" ]) + assert node.query("SHOW CREATE POLICIES ON mydb.*") == TSV([ "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default" ]) + assert node.query("SHOW CREATE POLICIES") == TSV([ "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default" ]) copy_policy_xml('all_rows.xml') assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 9c8b116e6d2..939c7d93e2d 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -151,6 +151,11 @@ def test_show_profiles(): instance.query("CREATE SETTINGS PROFILE xyz") assert instance.query("SHOW SETTINGS PROFILES") == "default\nreadonly\nxyz\n" assert instance.query("SHOW PROFILES") == "default\nreadonly\nxyz\n" + assert instance.query("SHOW CREATE PROFILE xyz") == "CREATE SETTINGS PROFILE xyz\n" + assert instance.query("SHOW CREATE SETTINGS PROFILE default") == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n" + assert instance.query("SHOW CREATE PROFILES") == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n"\ + "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n"\ + "CREATE SETTINGS PROFILE xyz\n" def test_allow_introspection(): From a40885fa8469dc2aba457f3beb9e88710308942c Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 11 Jun 2020 02:08:37 +0300 Subject: [PATCH 241/329] Add new command SHOW ACCESS. --- src/Interpreters/InterpreterFactory.cpp | 6 ++ .../InterpreterShowAccessEntitiesQuery.cpp | 2 +- .../InterpreterShowAccessEntitiesQuery.h | 5 +- .../InterpreterShowAccessQuery.cpp | 89 +++++++++++++++++++ src/Interpreters/InterpreterShowAccessQuery.h | 36 ++++++++ ...InterpreterShowCreateAccessEntityQuery.cpp | 2 +- .../InterpreterShowCreateAccessEntityQuery.h | 5 +- src/Interpreters/InterpreterShowGrantsQuery.h | 3 + src/Interpreters/ya.make | 1 + src/Parsers/ASTShowAccessQuery.h | 17 ++++ src/Parsers/ParserQueryWithOutput.cpp | 3 + src/Parsers/ParserShowAccessQuery.h | 32 +++++++ .../test_create_user_and_login/test.py | 11 ++- tests/integration/test_quota/test.py | 3 + tests/integration/test_role/test.py | 10 ++- tests/integration/test_row_policy/test.py | 10 +++ .../integration/test_settings_profile/test.py | 6 ++ 17 files changed, 231 insertions(+), 10 deletions(-) create mode 100644 src/Interpreters/InterpreterShowAccessQuery.cpp create mode 100644 src/Interpreters/InterpreterShowAccessQuery.h create mode 100644 src/Parsers/ASTShowAccessQuery.h create mode 100644 src/Parsers/ParserShowAccessQuery.h diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index ccaa8fa4067..6d631c37428 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -231,6 +233,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, Context & { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp index 379580fe58a..009b9c580d3 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp @@ -18,7 +18,7 @@ using EntityType = IAccessEntity::Type; InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, Context & context_) - : query_ptr(query_ptr_), context(context_), ignore_quota(query_ptr->as().type == EntityType::QUOTA) + : query_ptr(query_ptr_), context(context_) { } diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.h b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h index 5e20bdfa231..8fcd70919ba 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.h +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h @@ -15,15 +15,14 @@ public: BlockIO execute() override; - bool ignoreQuota() const override { return ignore_quota; } - bool ignoreLimits() const override { return ignore_quota; } + bool ignoreQuota() const override { return true; } + bool ignoreLimits() const override { return true; } private: String getRewrittenQuery() const; ASTPtr query_ptr; Context & context; - bool ignore_quota = false; }; } diff --git a/src/Interpreters/InterpreterShowAccessQuery.cpp b/src/Interpreters/InterpreterShowAccessQuery.cpp new file mode 100644 index 00000000000..c9541b4f5bf --- /dev/null +++ b/src/Interpreters/InterpreterShowAccessQuery.cpp @@ -0,0 +1,89 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +using EntityType = IAccessEntity::Type; + + +BlockIO InterpreterShowAccessQuery::execute() +{ + BlockIO res; + res.in = executeImpl(); + return res; +} + + +BlockInputStreamPtr InterpreterShowAccessQuery::executeImpl() const +{ + /// Build a create query. + ASTs queries = getCreateAndGrantQueries(); + + /// Build the result column. + MutableColumnPtr column = ColumnString::create(); + std::stringstream ss; + for (const auto & query : queries) + { + ss.str(""); + formatAST(*query, ss, false, true); + column->insert(ss.str()); + } + + String desc = "ACCESS"; + return std::make_shared(Block{{std::move(column), std::make_shared(), desc}}); +} + + +std::vector InterpreterShowAccessQuery::getEntities() const +{ + const auto & access_control = context.getAccessControlManager(); + context.checkAccess(AccessType::SHOW_ACCESS); + + std::vector entities; + for (auto type : ext::range(EntityType::MAX)) + { + auto ids = access_control.findAll(type); + for (const auto & id : ids) + { + if (auto entity = access_control.tryRead(id)) + entities.push_back(entity); + } + } + + boost::range::sort(entities, IAccessEntity::LessByTypeAndName{}); + return entities; +} + + +ASTs InterpreterShowAccessQuery::getCreateAndGrantQueries() const +{ + auto entities = getEntities(); + const auto & access_control = context.getAccessControlManager(); + + ASTs create_queries, grant_queries; + for (const auto & entity : entities) + { + create_queries.push_back(InterpreterShowCreateAccessEntityQuery::getCreateQuery(*entity, access_control)); + if (entity->isTypeOf(EntityType::USER) || entity->isTypeOf(EntityType::USER)) + boost::range::push_back(grant_queries, InterpreterShowGrantsQuery::getGrantQueries(*entity, access_control)); + } + + ASTs result = std::move(create_queries); + boost::range::push_back(result, std::move(grant_queries)); + return result; +} + +} diff --git a/src/Interpreters/InterpreterShowAccessQuery.h b/src/Interpreters/InterpreterShowAccessQuery.h new file mode 100644 index 00000000000..eb548c56241 --- /dev/null +++ b/src/Interpreters/InterpreterShowAccessQuery.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; +struct IAccessEntity; +using AccessEntityPtr = std::shared_ptr; + +/** Return all queries for creating access entities and grants. + */ +class InterpreterShowAccessQuery : public IInterpreter +{ +public: + InterpreterShowAccessQuery(const ASTPtr & query_ptr_, Context & context_) + : query_ptr(query_ptr_), context(context_) {} + + BlockIO execute() override; + + bool ignoreQuota() const override { return true; } + bool ignoreLimits() const override { return true; } + +private: + BlockInputStreamPtr executeImpl() const; + ASTs getCreateAndGrantQueries() const; + std::vector getEntities() const; + + ASTPtr query_ptr; + Context & context; +}; + + +} diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 55c5d961ad8..8d5f27e116d 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -218,7 +218,7 @@ namespace InterpreterShowCreateAccessEntityQuery::InterpreterShowCreateAccessEntityQuery(const ASTPtr & query_ptr_, const Context & context_) - : query_ptr(query_ptr_), context(context_), ignore_quota(query_ptr->as().type == EntityType::QUOTA) + : query_ptr(query_ptr_), context(context_) { } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h index 12f427b5eb0..5bacbd42988 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h @@ -23,8 +23,8 @@ public: BlockIO execute() override; - bool ignoreQuota() const override { return ignore_quota; } - bool ignoreLimits() const override { return ignore_quota; } + bool ignoreQuota() const override { return true; } + bool ignoreLimits() const override { return true; } static ASTPtr getCreateQuery(const IAccessEntity & entity, const AccessControlManager & access_control); static ASTPtr getAttachQuery(const IAccessEntity & entity); @@ -37,7 +37,6 @@ private: ASTPtr query_ptr; const Context & context; - bool ignore_quota = false; }; diff --git a/src/Interpreters/InterpreterShowGrantsQuery.h b/src/Interpreters/InterpreterShowGrantsQuery.h index af6c3065192..f5dbd110fd0 100644 --- a/src/Interpreters/InterpreterShowGrantsQuery.h +++ b/src/Interpreters/InterpreterShowGrantsQuery.h @@ -23,6 +23,9 @@ public: static ASTs getGrantQueries(const IAccessEntity & user_or_role, const AccessControlManager & access_control); static ASTs getAttachGrantQueries(const IAccessEntity & user_or_role); + bool ignoreQuota() const override { return true; } + bool ignoreLimits() const override { return true; } + private: BlockInputStreamPtr executeImpl(); ASTs getGrantQueries() const; diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 29be5d3c216..2fd540ab609 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -85,6 +85,7 @@ SRCS( InterpreterSelectWithUnionQuery.cpp InterpreterSetQuery.cpp InterpreterSetRoleQuery.cpp + InterpreterShowAccessQuery.cpp InterpreterShowAccessEntitiesQuery.cpp InterpreterShowCreateAccessEntityQuery.cpp InterpreterShowCreateQuery.cpp diff --git a/src/Parsers/ASTShowAccessQuery.h b/src/Parsers/ASTShowAccessQuery.h new file mode 100644 index 00000000000..dffd7ff2403 --- /dev/null +++ b/src/Parsers/ASTShowAccessQuery.h @@ -0,0 +1,17 @@ +#pragma once + +#include + + +namespace DB +{ + +struct ASTShowAccessQueryNames +{ + static constexpr auto ID = "ShowAccessQuery"; + static constexpr auto Query = "SHOW ACCESS"; +}; + +using ASTShowAccessQuery = ASTQueryWithOutputImpl; + +} diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp index a81305c0557..c7a42b5bdad 100644 --- a/src/Parsers/ParserQueryWithOutput.cpp +++ b/src/Parsers/ParserQueryWithOutput.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserOptimizeQuery optimize_p; ParserKillQueryQuery kill_query_p; ParserWatchQuery watch_p; + ParserShowAccessQuery show_access_p; ParserShowAccessEntitiesQuery show_access_entities_p; ParserShowCreateAccessEntityQuery show_create_access_entity_p; ParserShowGrantsQuery show_grants_p; @@ -70,6 +72,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec || kill_query_p.parse(pos, query, expected) || optimize_p.parse(pos, query, expected) || watch_p.parse(pos, query, expected) + || show_access_p.parse(pos, query, expected) || show_access_entities_p.parse(pos, query, expected) || show_grants_p.parse(pos, query, expected) || show_privileges_p.parse(pos, query, expected); diff --git a/src/Parsers/ParserShowAccessQuery.h b/src/Parsers/ParserShowAccessQuery.h new file mode 100644 index 00000000000..b6483aa3d43 --- /dev/null +++ b/src/Parsers/ParserShowAccessQuery.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +/** Query SHOW ACCESS + */ +class ParserShowAccessQuery : public IParserBase +{ +protected: + const char * getName() const override { return "SHOW ACCESS query"; } + + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override + { + auto query = std::make_shared(); + + if (!ParserKeyword("SHOW ACCESS").ignore(pos, expected)) + return false; + + node = query; + + return true; + } +}; + +} diff --git a/tests/integration/test_create_user_and_login/test.py b/tests/integration/test_create_user_and_login/test.py index 8cfc1c2c9dd..ddb3e57c63b 100644 --- a/tests/integration/test_create_user_and_login/test.py +++ b/tests/integration/test_create_user_and_login/test.py @@ -79,10 +79,19 @@ def test_introspection(): assert instance.query("SHOW GRANTS FOR A,B") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) assert instance.query("SHOW GRANTS FOR B,A") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) assert instance.query("SHOW GRANTS FOR ALL") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT ALL ON *.* TO default WITH GRANT OPTION" ]) - + assert instance.query("SHOW GRANTS", user='A') == TSV([ "GRANT SELECT ON test.table TO A" ]) assert instance.query("SHOW GRANTS", user='B') == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) + expected_access1 = "CREATE USER A\n"\ + "CREATE USER B\n"\ + "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" + expected_access2 = "GRANT SELECT ON test.table TO A\n"\ + "GRANT CREATE ON *.* TO B WITH GRANT OPTION\n"\ + "GRANT ALL ON *.* TO default WITH GRANT OPTION\n" + assert expected_access1 in instance.query("SHOW ACCESS") + assert expected_access2 in instance.query("SHOW ACCESS") + assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ TSV([[ "A", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], [ "B", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py index b9a6d80e54e..27aa353b9b1 100644 --- a/tests/integration/test_quota/test.py +++ b/tests/integration/test_quota/test.py @@ -215,6 +215,9 @@ def test_dcl_introspection(): assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", instance.query("SHOW QUOTA")) + expected_access = "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" + assert expected_access in instance.query("SHOW ACCESS") + # Add interval. copy_quota_xml('two_intervals.xml') assert instance.query("SHOW QUOTAS") == "myQuota\n" diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py index 6bae3baafee..e668b461389 100644 --- a/tests/integration/test_role/test.py +++ b/tests/integration/test_role/test.py @@ -118,7 +118,7 @@ def test_introspection(): assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION" ]) assert instance.query("SHOW GRANTS FOR R1") == "" assert instance.query("SHOW GRANTS FOR R2") == TSV([ "GRANT SELECT ON test.table TO R2", "REVOKE SELECT(x) ON test.table FROM R2" ]) - + assert instance.query("SHOW GRANTS", user='A') == TSV([ "GRANT SELECT ON test.table TO A", "GRANT R1 TO A" ]) assert instance.query("SHOW GRANTS", user='B') == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION" ]) assert instance.query("SHOW CURRENT ROLES", user='A') == TSV([[ "R1", 0, 1 ]]) @@ -126,6 +126,14 @@ def test_introspection(): assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([[ "R1", 0, 1, 1 ]]) assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([[ "R2", 1, 1, 1 ]]) + expected_access1 = "CREATE ROLE R1\n"\ + "CREATE ROLE R2\n" + expected_access2 = "GRANT R1 TO A\n" + expected_access3 = "GRANT R2 TO B WITH ADMIN OPTION" + assert expected_access1 in instance.query("SHOW ACCESS") + assert expected_access2 in instance.query("SHOW ACCESS") + assert expected_access3 in instance.query("SHOW ACCESS") + assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") ==\ TSV([[ "R1", "disk" ], [ "R2", "disk" ]]) diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index d5fbae654a8..aca3100196b 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -241,6 +241,16 @@ def test_dcl_introspection(): assert node.query("SHOW CREATE POLICIES ON mydb.*") == TSV([ "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default" ]) assert node.query("SHOW CREATE POLICIES") == TSV([ "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default" ]) + expected_access = "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another\n"\ + "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another\n"\ + "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another\n"\ + "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another\n"\ + "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n"\ + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n"\ + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n"\ + "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" + assert expected_access in node.query("SHOW ACCESS") + copy_policy_xml('all_rows.xml') assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING 1 TO default\n" diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 939c7d93e2d..d722717f2a7 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -151,11 +151,17 @@ def test_show_profiles(): instance.query("CREATE SETTINGS PROFILE xyz") assert instance.query("SHOW SETTINGS PROFILES") == "default\nreadonly\nxyz\n" assert instance.query("SHOW PROFILES") == "default\nreadonly\nxyz\n" + assert instance.query("SHOW CREATE PROFILE xyz") == "CREATE SETTINGS PROFILE xyz\n" assert instance.query("SHOW CREATE SETTINGS PROFILE default") == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n" assert instance.query("SHOW CREATE PROFILES") == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n"\ "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n"\ "CREATE SETTINGS PROFILE xyz\n" + + expected_access = "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n"\ + "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n"\ + "CREATE SETTINGS PROFILE xyz\n" + assert expected_access in instance.query("SHOW ACCESS") def test_allow_introspection(): From bd17aa8a0eb5dcca23034194672e57b1f85df048 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Jun 2020 22:11:17 +0300 Subject: [PATCH 242/329] Increase timeout in test --- .../0_stateless/01076_parallel_alter_replicated_zookeeper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index 15bb851fc77..4bb8da3d43c 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -105,7 +105,7 @@ sleep 1 counter=0 while [[ $($CLICKHOUSE_CLIENT --query "select * from system.mutations where table like 'concurrent_mutate_mt_%' and is_done=0" 2>&1) ]]; do - if [ "$counter" -gt 40 ] + if [ "$counter" -gt 120 ] then break fi From d10109dc38d6f2caa7b1e1a897e0a73082412bde Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 22:18:33 +0300 Subject: [PATCH 243/329] Remove excessive statement #11131 --- src/Storages/MergeTree/MergeTreeDataWriter.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 52eace30657..f3da98f0ba3 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -139,7 +139,6 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block return result; data.check(block, true); - block.checkNumberOfRows(); if (!data.hasPartitionKey()) /// Table is not partitioned. { From 1976b10ee0c73de553df12daab06aa413c61213c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 23:01:58 +0300 Subject: [PATCH 244/329] Proper query formatting in logs #3853 --- src/Interpreters/executeQuery.cpp | 33 +++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index e1e2108c0fc..91c72fa04c5 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include @@ -41,7 +43,6 @@ #include #include #include -#include namespace ProfileEvents @@ -70,11 +71,35 @@ static void checkASTSizeLimits(const IAST & ast, const Settings & settings) ast.checkSize(settings.max_ast_elements); } -/// NOTE This is wrong in case of single-line comments and in case of multiline string literals. + static String joinLines(const String & query) { - String res = query; - std::replace(res.begin(), res.end(), '\n', ' '); + /// Care should be taken. We don't join lines inside non-whitespace tokens (e.g. multiline string literals) + /// and we don't join line after single-line comment. + /// All other whitespaces replaced to a single whitespace. + + String res; + const char * begin = query.data(); + const char * end = begin + query.size(); + + Lexer lexer(begin, end); + Token token = lexer.nextToken(); + for (; !token.isEnd(); token = lexer.nextToken()) + { + if (token.type == TokenType::Whitespace) + { + res += ' '; + } + else if (token.type == TokenType::Comment) + { + res.append(token.begin, token.end); + if (token.end < end && *token.end == '\n') + res += '\n'; + } + else + res.append(token.begin, token.end); + } + return res; } From 511fa106afad60bdf7b06c15a13e23b108a67531 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 23:05:21 +0300 Subject: [PATCH 245/329] Better comment --- src/Interpreters/executeQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 91c72fa04c5..cff020e62f6 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -75,7 +75,7 @@ static void checkASTSizeLimits(const IAST & ast, const Settings & settings) static String joinLines(const String & query) { /// Care should be taken. We don't join lines inside non-whitespace tokens (e.g. multiline string literals) - /// and we don't join line after single-line comment. + /// and we don't join line after comment (because it can be single-line comment). /// All other whitespaces replaced to a single whitespace. String res; From 80bcaaacf27b831d2e5e29647832293acf2ad9de Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 23:07:39 +0300 Subject: [PATCH 246/329] Don't rewrite query in client when there are no query parameters --- programs/client/Client.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 63467c1129d..e396ae9c868 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -986,7 +986,10 @@ private: /// Process the query that doesn't require transferring data blocks to the server. void processOrdinaryQuery() { - /// We will always rewrite query (even if there are no query_parameters) because it will help to find errors in query formatter. + /// Rewrite query only when we have query parameters. + /// Note that if query is rewritten, comments in query are lost. + /// But the user often wants to see comments in server logs, query log, processlist, etc. + if (!query_parameters.empty()) { /// Replace ASTQueryParameter with ASTLiteral for prepared statements. ReplaceQueryParameterVisitor visitor(query_parameters); From ba8244198974856b09fd9100b9a65fcfb1dc009f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 23:13:57 +0300 Subject: [PATCH 247/329] Added a test --- .../01319_query_formatting_in_server_log.reference | 4 ++++ .../0_stateless/01319_query_formatting_in_server_log.sql | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/01319_query_formatting_in_server_log.reference create mode 100644 tests/queries/0_stateless/01319_query_formatting_in_server_log.sql diff --git a/tests/queries/0_stateless/01319_query_formatting_in_server_log.reference b/tests/queries/0_stateless/01319_query_formatting_in_server_log.reference new file mode 100644 index 00000000000..5fb3cc3a599 --- /dev/null +++ b/tests/queries/0_stateless/01319_query_formatting_in_server_log.reference @@ -0,0 +1,4 @@ +ab\ncd 1 +SeLeCt 'ab +cd' /* hello */ -- world + , 1; diff --git a/tests/queries/0_stateless/01319_query_formatting_in_server_log.sql b/tests/queries/0_stateless/01319_query_formatting_in_server_log.sql new file mode 100644 index 00000000000..dc88d3d48f7 --- /dev/null +++ b/tests/queries/0_stateless/01319_query_formatting_in_server_log.sql @@ -0,0 +1,6 @@ +SeLeCt 'ab +cd' /* hello */ -- world +, 1; + +SYSTEM FLUSH LOGS; +SELECT extract(message, 'SeL.+?;') FROM system.text_log WHERE event_date >= yesterday() AND message LIKE '%SeLeCt \'ab\n%' ORDER BY event_time DESC LIMIT 1 FORMAT TSVRaw; From bb6c0743fc512a817b4ee53dc47c53822083b10d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jun 2020 23:30:36 +0300 Subject: [PATCH 248/329] Change the level of log message about failure to listen, to warning #4406 --- programs/server/Server.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 9734bafe30e..25d8e5595b7 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -869,7 +869,7 @@ int Server::main(const std::vector & /*args*/) if (listen_try) { - LOG_ERROR(log, "{}. If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " + LOG_WARNING(log, "{}. If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " "specify not disabled IPv4 or IPv6 address to listen in element of configuration " "file. Example for disabled IPv6: 0.0.0.0 ." " Example for disabled IPv4: ::", @@ -1013,7 +1013,8 @@ int Server::main(const std::vector & /*args*/) } if (servers.empty()) - throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", ErrorCodes::NO_ELEMENTS_IN_CONFIG); + throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", + ErrorCodes::NO_ELEMENTS_IN_CONFIG); global_context->enableNamedSessions(); From bc58e22c5bcf85dc42b97af40357483f62a4ecf6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 01:23:13 +0300 Subject: [PATCH 249/329] Whitespace --- src/Common/XDBCBridgeHelper.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/XDBCBridgeHelper.h b/src/Common/XDBCBridgeHelper.h index 1609737107e..233c5c83df4 100644 --- a/src/Common/XDBCBridgeHelper.h +++ b/src/Common/XDBCBridgeHelper.h @@ -272,7 +272,8 @@ struct ODBCBridgeMixin return AccessType::ODBC; } - static std::unique_ptr startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout) + static std::unique_ptr startBridge( + const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout) { /// Path to executable folder Poco::Path path{config.getString("application.dir", "/usr/bin")}; From 1e73a56a778d2f2e864f8e932d18e1b44f2beba5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 01:23:56 +0300 Subject: [PATCH 250/329] Whitespace --- src/Common/XDBCBridgeHelper.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/XDBCBridgeHelper.h b/src/Common/XDBCBridgeHelper.h index 233c5c83df4..9320122d2e5 100644 --- a/src/Common/XDBCBridgeHelper.h +++ b/src/Common/XDBCBridgeHelper.h @@ -278,7 +278,6 @@ struct ODBCBridgeMixin /// Path to executable folder Poco::Path path{config.getString("application.dir", "/usr/bin")}; - std::vector cmd_args; path.setFileName("clickhouse-odbc-bridge"); From eabbabed04aba337e94d32b4a869bc84e883fbec Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 16 Jun 2020 01:24:00 +0300 Subject: [PATCH 251/329] fix 'LIMIT WITH TIES' with aliases --- src/Interpreters/InterpreterSelectQuery.cpp | 8 ++++++ .../01142_with_ties_and_aliases.reference | 25 +++++++++++++++++++ .../01142_with_ties_and_aliases.sql | 12 +++++++++ 3 files changed, 45 insertions(+) create mode 100644 tests/queries/0_stateless/01142_with_ties_and_aliases.reference create mode 100644 tests/queries/0_stateless/01142_with_ties_and_aliases.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ac17a3042d8..523e467261b 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -973,6 +973,14 @@ void InterpreterSelectQuery::executeImpl(QueryPipeline & pipeline, const BlockIn executeWithFill(pipeline); + /// If we have 'WITH TIES', we need execute limit before projection, + /// because in that case columns from 'ORDER BY' are used. + if (query.limit_with_ties) + { + executeLimit(pipeline); + has_prelimit = true; + } + /** We must do projection after DISTINCT because projection may remove some columns. */ executeProjection(pipeline, expressions.final_projection); diff --git a/tests/queries/0_stateless/01142_with_ties_and_aliases.reference b/tests/queries/0_stateless/01142_with_ties_and_aliases.reference new file mode 100644 index 00000000000..1846e07a908 --- /dev/null +++ b/tests/queries/0_stateless/01142_with_ties_and_aliases.reference @@ -0,0 +1,25 @@ +0 0 +1 0 +2 0 +3 0 +4 0 +1 +1 +1 +1 +1 +0 +1 +2 +3 +4 +0 0 +0 1 +0 2 +0 3 +0 4 +0 0 +0 1 +0 2 +0 3 +0 4 diff --git a/tests/queries/0_stateless/01142_with_ties_and_aliases.sql b/tests/queries/0_stateless/01142_with_ties_and_aliases.sql new file mode 100644 index 00000000000..f086cb9d907 --- /dev/null +++ b/tests/queries/0_stateless/01142_with_ties_and_aliases.sql @@ -0,0 +1,12 @@ +select number, intDiv(number,5) value from numbers(20) order by value limit 3 with ties; + +drop table if exists wt; +create table wt (a Int, b Int) engine = Memory; +insert into wt select 0, number from numbers(5); + +select 1 from wt order by a limit 3 with ties; +select b from wt order by a limit 3 with ties; +with a * 2 as c select a, b from wt order by c limit 3 with ties; +select a * 2 as c, b from wt order by c limit 3 with ties; + +drop table if exists wt; From 186d336e3199486f753b334d78d1ddad7c8b6fe6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 01:35:15 +0300 Subject: [PATCH 252/329] Use proper timeouts when communicating with xdbc-bridge --- src/Common/XDBCBridgeHelper.h | 6 ++++-- src/IO/ReadWriteBufferFromHTTP.h | 3 ++- src/Storages/StorageXDBC.cpp | 1 - src/TableFunctions/ITableFunctionXDBC.cpp | 6 +++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Common/XDBCBridgeHelper.h b/src/Common/XDBCBridgeHelper.h index 9320122d2e5..370a58498a5 100644 --- a/src/Common/XDBCBridgeHelper.h +++ b/src/Common/XDBCBridgeHelper.h @@ -109,7 +109,8 @@ public: uri.setPath(IDENTIFIER_QUOTE_HANDLER); uri.addQueryParameter("connection_string", getConnectionString()); - ReadWriteBufferFromHTTP buf(uri, Poco::Net::HTTPRequest::HTTP_POST, nullptr); + ReadWriteBufferFromHTTP buf( + uri, Poco::Net::HTTPRequest::HTTP_POST, {}, ConnectionTimeouts(http_timeout, http_timeout, http_timeout)); std::string character; readStringBinary(character, buf); if (character.length() > 1) @@ -208,7 +209,8 @@ private: { try { - ReadWriteBufferFromHTTP buf(ping_url, Poco::Net::HTTPRequest::HTTP_GET, nullptr); + ReadWriteBufferFromHTTP buf( + ping_url, Poco::Net::HTTPRequest::HTTP_GET, {}, ConnectionTimeouts(http_timeout, http_timeout, http_timeout)); return checkString(XDBCBridgeHelper::PING_OK_ANSWER, buf); } catch (...) diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index edd0b7f1579..2dc053dfa00 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -156,7 +156,8 @@ namespace detail public: using OutStreamCallback = std::function; - explicit ReadWriteBufferFromHTTPBase(UpdatableSessionPtr session_, + explicit ReadWriteBufferFromHTTPBase( + UpdatableSessionPtr session_, Poco::URI uri_, const std::string & method_ = {}, OutStreamCallback out_stream_callback_ = {}, diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index 08538798389..c090ca44034 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/src/TableFunctions/ITableFunctionXDBC.cpp b/src/TableFunctions/ITableFunctionXDBC.cpp index adf0c9240bc..f25e010ddbc 100644 --- a/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/src/TableFunctions/ITableFunctionXDBC.cpp @@ -75,7 +75,11 @@ StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Co columns_info_uri.addQueryParameter("external_table_functions_use_nulls", Poco::NumberFormatter::format(use_nulls)); - ReadWriteBufferFromHTTP buf(columns_info_uri, Poco::Net::HTTPRequest::HTTP_POST, nullptr); + ReadWriteBufferFromHTTP buf(columns_info_uri, Poco::Net::HTTPRequest::HTTP_POST, {}, + ConnectionTimeouts( + context.getSettingsRef().http_connection_timeout, + context.getSettingsRef().http_send_timeout, + context.getSettingsRef().http_receive_timeout)); std::string columns_info; readStringBinary(columns_info, buf); From cf0bd501e506ac85a025c948642941be8ccdf5ca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 01:36:12 +0300 Subject: [PATCH 253/329] Remove harmful default values from code --- src/IO/ReadWriteBufferFromHTTP.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 2dc053dfa00..40057e1f80e 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -246,9 +246,9 @@ class ReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase Date: Tue, 16 Jun 2020 01:54:19 +0300 Subject: [PATCH 254/329] Fix race condition in SYSTEM SYNC REPLICA --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 8a9dbceba04..6325b1adca4 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -646,7 +646,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C } } - if (some_active_mutations_were_killed) + if (some_active_mutations_were_killed && storage.queue_task_handle) storage.queue_task_handle->signalReadyToRun(); if (!entries_to_load.empty()) @@ -759,7 +759,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( LOG_DEBUG(log, "Removed mutation {} from local state.", entry->znode_name); } - if (mutation_was_active) + if (mutation_was_active && storage.queue_task_handle) storage.queue_task_handle->signalReadyToRun(); return entry; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 5931bca17ea..885db89e5b0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5643,9 +5643,16 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI /// Let's fetch new log entries firstly queue.pullLogsToQueue(getZooKeeper()); - /// This is significant, because the execution of this task could be delayed at BackgroundPool. - /// And we force it to be executed. - queue_task_handle->signalReadyToRun(); + + { + auto lock = queue.lockQueue(); + if (!queue_task_handle) + return false; + + /// This is significant, because the execution of this task could be delayed at BackgroundPool. + /// And we force it to be executed. + queue_task_handle->signalReadyToRun(); + } Poco::Event target_size_event; auto callback = [&target_size_event, queue_size] (size_t new_queue_size) From 00224ee94f482573ca996cecc11a8110aba8dc15 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 02:04:12 +0300 Subject: [PATCH 255/329] Added a test --- ...01320_create_sync_race_condition.reference | 0 .../01320_create_sync_race_condition.sh | 28 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/queries/0_stateless/01320_create_sync_race_condition.reference create mode 100755 tests/queries/0_stateless/01320_create_sync_race_condition.sh diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition.reference b/tests/queries/0_stateless/01320_create_sync_race_condition.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition.sh b/tests/queries/0_stateless/01320_create_sync_race_condition.sh new file mode 100755 index 00000000000..2e42033644a --- /dev/null +++ b/tests/queries/0_stateless/01320_create_sync_race_condition.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS r;" + +function thread1() +{ + while true; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE r (x UInt64) ENGINE = ReplicatedMergeTree('/test/table', 'r') ORDER BY x; DROP TABLE r;"; done +} + +function thread2() +{ + while true; do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA r" 2>/dev/null; done +} + +export -f thread1 +export -f thread2 + +timeout 10 bash -c thread1 & +timeout 10 bash -c thread2 & + +wait + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS r;" From e0f4c64acf8f492ed7d62de1a7c65fa520c7a333 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 02:18:15 +0300 Subject: [PATCH 256/329] Fix flaky test --- .../01064_incremental_streaming_from_2_src_with_feedback.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql b/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql index 984f897ec2a..c2d0333bf46 100644 --- a/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql +++ b/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql @@ -1,7 +1,5 @@ SET joined_subquery_requires_alias = 0; -SYSTEM STOP MERGES; - -- incremental streaming usecase -- that has sense only if data filling order has guarantees of chronological order @@ -77,6 +75,8 @@ AS LEFT JOIN (SELECT id, maxMerge(latest_login_time) as current_latest_login_time FROM target_table WHERE id IN (SELECT id FROM checkouts) GROUP BY id) USING (id) GROUP BY id; +-- This query has effect only for existing tables, so it must be located after CREATE. +SYSTEM STOP MERGES; -- feed with some initial values INSERT INTO logins SELECT number as id, '2000-01-01 08:00:00' from numbers(50000); From a753ba9c6a2ac4502570123e6cb391fd70595a9e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 02:24:32 +0300 Subject: [PATCH 257/329] Checked and corrected all other tests --- .../0_stateless/00446_clear_column_in_partition_zookeeper.sql | 2 +- .../0_stateless/00653_verification_monotonic_data_load.sh | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql b/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql index 861c768edc9..996c84903a9 100644 --- a/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql +++ b/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql @@ -21,12 +21,12 @@ DROP TABLE clear_column; SELECT '===Replicated case==='; -SYSTEM STOP MERGES; DROP TABLE IF EXISTS clear_column1 NO DELAY; DROP TABLE IF EXISTS clear_column2 NO DELAY; SELECT sleep(1) FORMAT Null; CREATE TABLE clear_column1 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/clear_column', '1', d, d, 8192); CREATE TABLE clear_column2 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/clear_column', '2', d, d, 8192); +SYSTEM STOP MERGES; INSERT INTO clear_column1 (d) VALUES ('2000-01-01'), ('2000-02-01'); SYSTEM SYNC REPLICA clear_column2; diff --git a/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh b/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh index 69c0db567df..e52610f03ba 100755 --- a/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh +++ b/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh @@ -13,8 +13,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -${CLICKHOUSE_CLIENT} --query="SYSTEM STOP MERGES;" - ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS string_test_table;" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS fixed_string_test_table;" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS signed_integer_test_table;" @@ -29,6 +27,7 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE unsigned_integer_test_table (val UInt ${CLICKHOUSE_CLIENT} --query="CREATE TABLE enum_test_table (val Enum16('hello' = 1, 'world' = 2, 'yandex' = 256, 'clickhouse' = 257)) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0;" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE date_test_table (val Date) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0;" +${CLICKHOUSE_CLIENT} --query="SYSTEM STOP MERGES;" ${CLICKHOUSE_CLIENT} --query="INSERT INTO string_test_table VALUES ('0'), ('2'), ('2');" ${CLICKHOUSE_CLIENT} --query="INSERT INTO fixed_string_test_table VALUES ('0'), ('2'), ('2');" From d08736f7083bdc948d98b1a3ae7ff418505d8e58 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 02:27:22 +0300 Subject: [PATCH 258/329] Added a comment --- src/Interpreters/InterpreterSystemQuery.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Interpreters/InterpreterSystemQuery.h b/src/Interpreters/InterpreterSystemQuery.h index de2291746d9..b55d1bda09b 100644 --- a/src/Interpreters/InterpreterSystemQuery.h +++ b/src/Interpreters/InterpreterSystemQuery.h @@ -16,6 +16,19 @@ class Context; class AccessRightsElements; class ASTSystemQuery; + +/** Implement various SYSTEM queries. + * Examples: SYSTEM SHUTDOWN, SYSTEM DROP MARK CACHE. + * + * Some commands are intended to stop/start background actions for tables and comes with two variants: + * + * 1. SYSTEM STOP MERGES table, SYSTEM START MERGES table + * - start/stop actions for specific table. + * + * 2. SYSTEM STOP MERGES, SYSTEM START MERGES + * - start/stop actions for all existing tables. + * Note that the actions for tables that will be created after this query will not be affected. + */ class InterpreterSystemQuery : public IInterpreter { public: From 85a7676bf4ed136f8f99ac9ebe2b905c8346ae80 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 02:32:30 +0300 Subject: [PATCH 259/329] Fix flaky unit tests with ZooKeeper. --- .../gtest_zkutil_test_multi_exception.cpp | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp b/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp index 8440b4fe7c9..19104e02e24 100644 --- a/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp +++ b/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp @@ -14,15 +14,27 @@ using namespace DB; TEST(zkutil, ZookeeperConnected) { - try + /// In our CI infrastructure it is typical that ZooKeeper is unavailable for some amount of time. + size_t i; + for (i = 0; i < 100; ++i) { - auto zookeeper = std::make_unique("localhost:2181"); - zookeeper->exists("/"); - zookeeper->createIfNotExists("/clickhouse_test", "Unit tests of ClickHouse"); + try + { + auto zookeeper = std::make_unique("localhost:2181"); + zookeeper->exists("/"); + zookeeper->createIfNotExists("/clickhouse_test", "Unit tests of ClickHouse"); + } + catch (...) + { + std::cerr << "Zookeeper is unavailable, try " << i << std::endl; + sleep(1); + continue; + } + break; } - catch (...) + if (i == 100) { - std::cerr << "No zookeeper. skip tests." << std::endl; + std::cerr << "No zookeeper after " << i << " tries. skip tests." << std::endl; exit(0); } } From 97d5897d2b5d79ed5d4901735b4f9cae7247d3c6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 02:42:10 +0300 Subject: [PATCH 260/329] Fix test --- tests/queries/0_stateless/01091_num_threads.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01091_num_threads.sql b/tests/queries/0_stateless/01091_num_threads.sql index 876a2d15d1a..dc397d62305 100644 --- a/tests/queries/0_stateless/01091_num_threads.sql +++ b/tests/queries/0_stateless/01091_num_threads.sql @@ -4,7 +4,7 @@ set log_query_threads=1; SELECT 1; SYSTEM FLUSH LOGS; -WITH +WITH ( SELECT query_id FROM system.query_log @@ -19,11 +19,11 @@ WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master select sum(number) from numbers(1000000); SYSTEM FLUSH LOGS; -WITH +WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT sum(number) FROM numbers(1000000)') AND (event_date >= (today() - 1)) + WHERE (query LIKE 'select sum(number) from numbers(1000000);%') AND (event_date >= (today() - 1)) ORDER BY event_time DESC LIMIT 1 ) AS id @@ -34,11 +34,11 @@ WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master select sum(number) from numbers_mt(1000000); SYSTEM FLUSH LOGS; -WITH +WITH ( SELECT query_id FROM system.query_log - WHERE (query = 'SELECT sum(number) FROM numbers_mt(1000000)') AND (event_date >= (today() - 1)) + WHERE (query LIKE 'select sum(number) from numbers_mt(1000000);%') AND (event_date >= (today() - 1)) ORDER BY event_time DESC LIMIT 1 ) AS id From c85b9ae151391a9bc8294777164d9e716eeddbd4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 02:43:05 +0300 Subject: [PATCH 261/329] Fix test --- .../0_stateless/01070_exception_code_in_query_log_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql b/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql index 2c99ba54112..b9627a0f8a8 100644 --- a/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql +++ b/tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql @@ -3,5 +3,5 @@ SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; -- { serve CREATE TABLE test_table_for_01070_exception_code_in_query_log_table (value UInt64) ENGINE=Memory(); SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; SYSTEM FLUSH LOGS; -SELECT exception_code FROM system.query_log WHERE query = 'SELECT * FROM test_table_for_01070_exception_code_in_query_log_table' AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE ORDER BY exception_code; +SELECT exception_code FROM system.query_log WHERE lower(query) LIKE lower('SELECT * FROM test_table_for_01070_exception_code_in_query_log_table%') AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE ORDER BY exception_code; DROP TABLE IF EXISTS test_table_for_01070_exception_code_in_query_log_table; From edff54e3abb7b7f7d1baa47b8b13d1f02be379d7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 04:08:01 +0300 Subject: [PATCH 262/329] Maybe fix max_parser_depth test --- tests/queries/0_stateless/01196_max_parser_depth.sh | 8 +++++--- tests/queries/shell_config.sh | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/01196_max_parser_depth.sh b/tests/queries/0_stateless/01196_max_parser_depth.sh index 471c1c22ecb..24c219b2241 100755 --- a/tests/queries/0_stateless/01196_max_parser_depth.sh +++ b/tests/queries/0_stateless/01196_max_parser_depth.sh @@ -3,6 +3,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -{ printf "select "; for x in {1..1000}; do printf "coalesce(null, "; done; printf "1"; for x in {1..1000}; do printf ")"; done; } | $CLICKHOUSE_CLIENT 2>&1 | grep -o -F 'Code: 306' -{ printf "select "; for x in {1..1000}; do printf "coalesce(null, "; done; printf "1"; for x in {1..1000}; do printf ")"; done; } | $CLICKHOUSE_LOCAL 2>&1 | grep -o -F 'Code: 306' -{ printf "select "; for x in {1..1000}; do printf "coalesce(null, "; done; printf "1"; for x in {1..1000}; do printf ")"; done; } | $CLICKHOUSE_CURL --data-binary @- -vsS "$CLICKHOUSE_URL" 2>&1 | grep -o -F 'Code: 306' +{ printf "select "; for x in {1..1000}; do printf "coalesce(null, "; done; printf "1"; for x in {1..1000}; do printf ")"; done; } > ${CLICKHOUSE_TMP}/query + +cat ${CLICKHOUSE_TMP}/query | $CLICKHOUSE_CLIENT 2>&1 | grep -o -F 'Code: 306' +cat ${CLICKHOUSE_TMP}/query | $CLICKHOUSE_LOCAL 2>&1 | grep -o -F 'Code: 306' +cat ${CLICKHOUSE_TMP}/query | $CLICKHOUSE_CURL --data-binary @- -vsS "$CLICKHOUSE_URL" 2>&1 | grep -o -F 'Code: 306' diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 8d66a568524..8fe79bd3ccd 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -71,7 +71,7 @@ mkdir -p ${CLICKHOUSE_TMP} function clickhouse_client_removed_host_parameter() { - # removing only `--host=value` and `--host value` (removing '-hvalue' feels to dangerous) with python regex. - # bash regex magic is arcane, but version dependant and weak; sed or awk are not really portable. - $(echo "$CLICKHOUSE_CLIENT" | python -c "import sys, re; print re.sub('--host(\s+|=)[^\s]+', '', sys.stdin.read())") "$@" + # removing only `--host=value` and `--host value` (removing '-hvalue' feels to dangerous) with python regex. + # bash regex magic is arcane, but version dependant and weak; sed or awk are not really portable. + $(echo "$CLICKHOUSE_CLIENT" | python -c "import sys, re; print re.sub('--host(\s+|=)[^\s]+', '', sys.stdin.read())") "$@" } From 679c3f8ff335abfa54baae3726486ccf7bb53868 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 16 Jun 2020 04:50:22 +0300 Subject: [PATCH 263/329] Update 00446_clear_column_in_partition_zookeeper.sql --- .../0_stateless/00446_clear_column_in_partition_zookeeper.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql b/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql index 996c84903a9..8f6fe5340c9 100644 --- a/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql +++ b/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql @@ -26,7 +26,6 @@ DROP TABLE IF EXISTS clear_column2 NO DELAY; SELECT sleep(1) FORMAT Null; CREATE TABLE clear_column1 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/clear_column', '1', d, d, 8192); CREATE TABLE clear_column2 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/clear_column', '2', d, d, 8192); -SYSTEM STOP MERGES; INSERT INTO clear_column1 (d) VALUES ('2000-01-01'), ('2000-02-01'); SYSTEM SYNC REPLICA clear_column2; @@ -63,9 +62,7 @@ SELECT sum(data_uncompressed_bytes) FROM system.columns WHERE database=currentDa ALTER TABLE clear_column1 CLEAR COLUMN s IN PARTITION '200001'; ALTER TABLE clear_column1 CLEAR COLUMN s IN PARTITION '200002'; --- Merges cannot be blocked after all manipulations SET optimize_throw_if_noop = 1; -SYSTEM START MERGES; OPTIMIZE TABLE clear_column1 PARTITION '200001'; OPTIMIZE TABLE clear_column1 PARTITION '200002'; From 22a92faab649479c69bcc3eab7a7f6bfaef30c45 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 05:14:53 +0300 Subject: [PATCH 264/329] Avoid connection to replica when fetches are cancelled --- src/Storages/MergeTree/DataPartsExchange.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index acc3bf38461..6796e630ff2 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -63,8 +63,10 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo static std::atomic_uint total_sends {0}; - if ((data_settings->replicated_max_parallel_sends && total_sends >= data_settings->replicated_max_parallel_sends) - || (data_settings->replicated_max_parallel_sends_for_table && data.current_table_sends >= data_settings->replicated_max_parallel_sends_for_table)) + if ((data_settings->replicated_max_parallel_sends + && total_sends >= data_settings->replicated_max_parallel_sends) + || (data_settings->replicated_max_parallel_sends_for_table + && data.current_table_sends >= data_settings->replicated_max_parallel_sends_for_table)) { response.setStatus(std::to_string(HTTP_TOO_MANY_REQUESTS)); response.setReason("Too many concurrent fetches, try again later"); @@ -182,6 +184,9 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( bool to_detached, const String & tmp_prefix_) { + if (blocker.isCancelled()) + throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); + /// Validation of the input that may come from malicious replica. MergeTreePartInfo::fromPartName(part_name, data.format_version); const auto data_settings = data.getSettings(); @@ -294,7 +299,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart( if (blocker.isCancelled()) { - /// NOTE The is_cancelled flag also makes sense to check every time you read over the network, performing a poll with a not very large timeout. + /// NOTE The is_cancelled flag also makes sense to check every time you read over the network, + /// performing a poll with a not very large timeout. /// And now we check it only between read chunks (in the `copyData` function). disk->removeRecursive(part_download_path); throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); From 03058c3c25daf38468736f734c08f95a5b152121 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 16 Jun 2020 05:55:27 +0300 Subject: [PATCH 265/329] fix 'ORDER BY WITH FILL' over const columns --- .../Transforms/FillingTransform.cpp | 5 +++-- .../01145_with_fill_const.reference | 20 +++++++++++++++++++ .../0_stateless/01145_with_fill_const.sql | 6 ++++++ 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01145_with_fill_const.reference create mode 100644 tests/queries/0_stateless/01145_with_fill_const.sql diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index 50fac121819..e8d56389eac 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -107,8 +107,9 @@ void FillingTransform::transform(Chunk & chunk) { for (size_t pos : positions) { - new_columns.push_back(old_columns[pos]); - new_mutable_columns.push_back(old_columns[pos]->cloneEmpty()->assumeMutable()); + auto old_column = old_columns[pos]->convertToFullColumnIfConst(); + new_columns.push_back(old_column); + new_mutable_columns.push_back(old_column->cloneEmpty()->assumeMutable()); } }; diff --git a/tests/queries/0_stateless/01145_with_fill_const.reference b/tests/queries/0_stateless/01145_with_fill_const.reference new file mode 100644 index 00000000000..fa72c3c5993 --- /dev/null +++ b/tests/queries/0_stateless/01145_with_fill_const.reference @@ -0,0 +1,20 @@ +2020-06-16 00:00:00 +2020-06-16 00:30:00 +2020-06-16 01:00:00 +2020-06-16 01:30:00 +2020-06-16 02:00:00 +2020-06-16 02:30:00 +2020-06-16 03:00:00 +2020-06-16 03:30:00 +2020-06-16 04:00:00 +2020-06-16 04:30:00 +2020-06-16 05:00:00 +2020-06-16 05:30:00 +2020-06-16 06:00:00 +2020-06-16 06:30:00 +2020-06-16 07:00:00 +2020-06-16 07:30:00 +2020-06-16 08:00:00 +2020-06-16 08:30:00 +2020-06-16 09:00:00 +2020-06-16 09:30:00 diff --git a/tests/queries/0_stateless/01145_with_fill_const.sql b/tests/queries/0_stateless/01145_with_fill_const.sql new file mode 100644 index 00000000000..531d202c02a --- /dev/null +++ b/tests/queries/0_stateless/01145_with_fill_const.sql @@ -0,0 +1,6 @@ +WITH toDateTime('2020-06-16 03:00:00') AS date_time +SELECT date_time ORDER BY date_time ASC +WITH FILL + FROM toDateTime('2020-06-16 00:00:00') + TO toDateTime('2020-06-16 10:00:00') + STEP 1800; From bd330cfeb61dafed0c8c09e583235476eba7c279 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jun 2020 05:56:50 +0300 Subject: [PATCH 266/329] Update test --- tests/integration/test_quorum_inserts/test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py index 27901842692..607fe93f1ef 100644 --- a/tests/integration/test_quorum_inserts/test.py +++ b/tests/integration/test_quorum_inserts/test.py @@ -284,11 +284,14 @@ def test_insert_quorum_with_ttl(started_cluster): zero.query("INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", settings={'insert_quorum_timeout' : 5000}) - - assert TSV("1\t2011-01-01\n") == TSV(first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency' : 0})) - assert TSV("1\t2011-01-01\n") == TSV(first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency' : 1})) - print("Inserts should resume.") zero.query("INSERT INTO test_insert_quorum_with_ttl(a, d) VALUES(2, '2012-02-02')") + first.query("OPTIMIZE TABLE test_insert_quorum_with_ttl") + first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") + zero.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") + + assert TSV("2\t2012-02-02\n") == TSV(first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency' : 0})) + assert TSV("2\t2012-02-02\n") == TSV(first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency' : 1})) + execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_ttl") From 9a26d48ad0e595468225566aafb8c78d41a20ae0 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 16 Jun 2020 09:31:00 +0300 Subject: [PATCH 267/329] Basic blog similar to docs (#11609) * Basic blog similar to docs * rename post * no post content in post_meta * update readme and template * more "en" content * complete "en" content * build blog redirects * redirects for migration * link sitemaps * update po * add "ru" content * ru redirects * remove old domain mentions * adjust styles * content improvements * +1 alt * use main images from CDN * use re-hosted in-content images * extra vertical margin around embedded youtube * minor improvements * adjust post page * adjust html meta * adjust post page * improve blog rendering --- docs/en/introduction/adopters.md | 2 +- .../sql-reference/data-types/domains/ipv4.md | 2 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- .../functions/array-functions.md | 6 +- docs/en/whats-new/changelog/2017.md | 2 +- docs/es/introduction/adopters.md | 2 +- .../sql-reference/data-types/domains/ipv4.md | 2 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- .../functions/array-functions.md | 6 +- docs/es/whats-new/changelog/2017.md | 2 +- docs/fa/introduction/adopters.md | 2 +- .../sql-reference/data-types/domains/ipv4.md | 2 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- .../functions/array-functions.md | 6 +- docs/fa/whats-new/changelog/2017.md | 2 +- docs/fr/introduction/adopters.md | 2 +- .../sql-reference/data-types/domains/ipv4.md | 2 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- .../functions/array-functions.md | 6 +- docs/fr/whats-new/changelog/2017.md | 2 +- docs/ja/introduction/adopters.md | 2 +- docs/ja/introduction/distinctive-features.md | 2 +- docs/ja/introduction/history.md | 2 +- docs/ja/introduction/performance.md | 6 +- .../sql-reference/data-types/domains/ipv4.md | 2 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- .../functions/array-functions.md | 6 +- docs/ja/whats-new/changelog/2017.md | 2 +- .../sql-reference/data-types/domains/ipv4.md | 2 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- .../functions/array-functions.md | 4 +- docs/tools/blog.py | 107 +++++++++++ docs/tools/build.py | 38 ++-- docs/tools/mdx_clickhouse.py | 28 ++- docs/tools/nav.py | 48 ++++- docs/tools/redirects.py | 38 ++-- docs/tools/website.py | 39 ++++ docs/tr/introduction/adopters.md | 2 +- .../sql-reference/data-types/domains/ipv4.md | 2 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- .../functions/array-functions.md | 6 +- docs/tr/whats-new/changelog/2017.md | 2 +- docs/zh/introduction/adopters.md | 2 +- .../sql-reference/data-types/domains/ipv4.md | 2 +- .../sql-reference/data-types/domains/ipv6.md | 2 +- docs/zh/whats-new/changelog/2017.md | 2 +- website/blog/README.md | 47 +++++ ...on-of-data-structures-in-yandex-metrica.md | 108 +++++++++++ .../2016/how-to-update-data-in-clickhouse.md | 169 ++++++++++++++++++ .../en/2016/yandex-opensources-clickhouse.md | 12 ++ .../en/2017/clickhouse-at-data-scale-2017.md | 10 ++ .../2017/clickhouse-at-percona-live-2017.md | 22 +++ ...ckhouse-meetup-in-berlin-october-5-2017.md | 10 ++ ...khouse-meetup-in-santa-clara-may-4-2017.md | 8 + .../join-the-clickhouse-meetup-in-berlin.md | 13 ++ ...ouse-meetup-in-amsterdam-on-november-15.md | 8 + .../2018/clickhouse-at-analysys-a10-2018.md | 27 +++ .../clickhouse-at-percona-live-europe-2018.md | 25 +++ ...ty-meetup-in-beijing-on-january-27-2018.md | 68 +++++++ ...ty-meetup-in-beijing-on-october-28-2018.md | 54 ++++++ ...mmunity-meetup-in-berlin-on-july-3-2018.md | 39 ++++ ...se-community-meetup-in-berlin-on-july-3.md | 8 + ...unity-meetup-in-paris-on-october-2-2018.md | 20 +++ ...meetup-in-amsterdam-on-november-15-2018.md | 27 +++ .../en/2018/concept-cloud-mergetree-tables.md | 120 +++++++++++++ .../2019/clickhouse-at-percona-live-2019.md | 38 ++++ ...nese-academy-of-science-on-june-11-2019.md | 17 ++ ...khouse-meetup-in-beijing-on-june-8-2019.md | 35 ++++ ...khouse-meetup-in-limassol-on-may-7-2019.md | 41 +++++ ...khouse-meetup-in-madrid-on-april-2-2019.md | 28 +++ ...-meetup-in-san-francisco-on-june-4-2019.md | 10 ++ ...peed-up-lz4-decompression-in-clickhouse.md | 12 ++ ...of-clickhouse-meetups-in-china-for-2019.md | 14 ++ .../five-methods-for-database-obfuscation.md | 10 ++ website/blog/en/index.md | 3 + website/blog/en/redirects.txt | 32 ++++ ...khouse-meetup-v-moskve-21-noyabrya-2016.md | 8 + .../ru/2016/clickhouse-na-highload-2016.md | 14 ++ ...raneniya-i-obrabotki-dannykh-v-yandekse.md | 10 ++ .../ru/2016/yandeks-otkryvaet-clickhouse.md | 10 ++ .../ru/2017/clickhouse-meetup-edet-v-minsk.md | 14 ++ ...use-meetup-v-ekaterinburge-16-maya-2017.md | 8 + .../2017/clickhouse-meetup-v-minske-itogi.md | 16 ++ ...se-meetup-v-novosibirske-3-aprelya-2017.md | 10 ++ ...tup-v-sankt-peterburge-28-fevralya-2017.md | 8 + .../blog/ru/2017/clickhouse-na-uwdc-2017.md | 10 ++ ...ickhouse-meetup-v-limassole-7-maya-2019.md | 38 ++++ ...house-meetup-v-moskve-5-sentyabrya-2019.md | 10 ++ ...se-meetup-v-novosibirske-26-iyunya-2019.md | 12 ++ ...eetup-v-sankt-peterburge-27-iyulya-2019.md | 10 ++ ...ickrouse-meetup-v-minske-11-iyulya-2019.md | 12 ++ website/blog/ru/index.md | 3 + website/blog/ru/redirects.txt | 15 ++ website/css/blog.css | 8 + website/locale/en/LC_MESSAGES/messages.po | 22 ++- website/locale/es/LC_MESSAGES/messages.mo | Bin 6641 -> 6739 bytes website/locale/es/LC_MESSAGES/messages.po | 22 ++- website/locale/fa/LC_MESSAGES/messages.mo | Bin 7453 -> 7553 bytes website/locale/fa/LC_MESSAGES/messages.po | 22 ++- website/locale/fr/LC_MESSAGES/messages.mo | Bin 6490 -> 6585 bytes website/locale/fr/LC_MESSAGES/messages.po | 22 ++- website/locale/ja/LC_MESSAGES/messages.mo | Bin 6613 -> 6691 bytes website/locale/ja/LC_MESSAGES/messages.po | 22 ++- website/locale/messages.pot | 22 ++- website/locale/ru/LC_MESSAGES/messages.mo | Bin 8556 -> 8672 bytes website/locale/ru/LC_MESSAGES/messages.po | 22 ++- website/locale/tr/LC_MESSAGES/messages.mo | Bin 6378 -> 6473 bytes website/locale/tr/LC_MESSAGES/messages.po | 22 ++- website/locale/zh/LC_MESSAGES/messages.mo | Bin 5926 -> 6007 bytes website/locale/zh/LC_MESSAGES/messages.po | 22 ++- website/main.html | 41 +++-- website/sitemap-index.xml | 6 + website/templates/blog/content.html | 43 +++++ website/templates/blog/footer.html | 9 + website/templates/blog/nav.html | 45 +++++ website/templates/common_meta.html | 11 +- website/templates/docs/ld_json.html | 13 +- website/templates/index/community.html | 4 +- website/templates/index/nav.html | 3 +- 119 files changed, 1848 insertions(+), 184 deletions(-) create mode 100644 docs/tools/blog.py create mode 100644 website/blog/README.md create mode 100644 website/blog/en/2016/evolution-of-data-structures-in-yandex-metrica.md create mode 100644 website/blog/en/2016/how-to-update-data-in-clickhouse.md create mode 100644 website/blog/en/2016/yandex-opensources-clickhouse.md create mode 100644 website/blog/en/2017/clickhouse-at-data-scale-2017.md create mode 100644 website/blog/en/2017/clickhouse-at-percona-live-2017.md create mode 100644 website/blog/en/2017/clickhouse-meetup-in-berlin-october-5-2017.md create mode 100644 website/blog/en/2017/clickhouse-meetup-in-santa-clara-may-4-2017.md create mode 100644 website/blog/en/2017/join-the-clickhouse-meetup-in-berlin.md create mode 100644 website/blog/en/2018/announcing-clickhouse-meetup-in-amsterdam-on-november-15.md create mode 100644 website/blog/en/2018/clickhouse-at-analysys-a10-2018.md create mode 100644 website/blog/en/2018/clickhouse-at-percona-live-europe-2018.md create mode 100644 website/blog/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018.md create mode 100644 website/blog/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018.md create mode 100644 website/blog/en/2018/clickhouse-community-meetup-in-berlin-on-july-3-2018.md create mode 100644 website/blog/en/2018/clickhouse-community-meetup-in-berlin-on-july-3.md create mode 100644 website/blog/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018.md create mode 100644 website/blog/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018.md create mode 100644 website/blog/en/2018/concept-cloud-mergetree-tables.md create mode 100644 website/blog/en/2019/clickhouse-at-percona-live-2019.md create mode 100644 website/blog/en/2019/clickhouse-lecture-at-institute-of-computing-technology-chinese-academy-of-science-on-june-11-2019.md create mode 100644 website/blog/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019.md create mode 100644 website/blog/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019.md create mode 100644 website/blog/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019.md create mode 100644 website/blog/en/2019/clickhouse-meetup-in-san-francisco-on-june-4-2019.md create mode 100644 website/blog/en/2019/how-to-speed-up-lz4-decompression-in-clickhouse.md create mode 100644 website/blog/en/2019/schedule-of-clickhouse-meetups-in-china-for-2019.md create mode 100644 website/blog/en/2020/five-methods-for-database-obfuscation.md create mode 100644 website/blog/en/index.md create mode 100644 website/blog/en/redirects.txt create mode 100644 website/blog/ru/2016/clickhouse-meetup-v-moskve-21-noyabrya-2016.md create mode 100644 website/blog/ru/2016/clickhouse-na-highload-2016.md create mode 100644 website/blog/ru/2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md create mode 100644 website/blog/ru/2016/yandeks-otkryvaet-clickhouse.md create mode 100644 website/blog/ru/2017/clickhouse-meetup-edet-v-minsk.md create mode 100644 website/blog/ru/2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017.md create mode 100644 website/blog/ru/2017/clickhouse-meetup-v-minske-itogi.md create mode 100644 website/blog/ru/2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017.md create mode 100644 website/blog/ru/2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md create mode 100644 website/blog/ru/2017/clickhouse-na-uwdc-2017.md create mode 100644 website/blog/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019.md create mode 100644 website/blog/ru/2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019.md create mode 100644 website/blog/ru/2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019.md create mode 100644 website/blog/ru/2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md create mode 100644 website/blog/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019.md create mode 100644 website/blog/ru/index.md create mode 100644 website/blog/ru/redirects.txt create mode 100644 website/css/blog.css create mode 100644 website/templates/blog/content.html create mode 100644 website/templates/blog/footer.html create mode 100644 website/templates/blog/nav.html diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 081f963f74f..df9cdfa6430 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -35,7 +35,7 @@ toc_title: Adopters | [Exness](https://www.exness.com){.favicon} | Trading | Metrics, Logging | — | — | [Talk in Russian, May 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | [Geniee](https://geniee.co.jp){.favicon} | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | | [HUYA](https://www.huya.com/){.favicon} | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| [Idealista](https://www.idealista.com){.favicon} | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| [Idealista](https://www.idealista.com){.favicon} | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.tech/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | | [Infovista](https://www.infovista.com/){.favicon} | Networks | Analytics | — | — | [Slides in English, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | | [InnoGames](https://www.innogames.com){.favicon} | Games | Metrics, Logging | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | | [Integros](https://integros.com){.favicon} | Platform for video services | Analytics | — | — | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | diff --git a/docs/en/sql-reference/data-types/domains/ipv4.md b/docs/en/sql-reference/data-types/domains/ipv4.md index d8735d70b29..1237514b9e7 100644 --- a/docs/en/sql-reference/data-types/domains/ipv4.md +++ b/docs/en/sql-reference/data-types/domains/ipv4.md @@ -31,7 +31,7 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from; `IPv4` domain supports custom input format as IPv4-strings: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.tech/docs/en/', '116.106.34.242'); SELECT * FROM hits; ``` diff --git a/docs/en/sql-reference/data-types/domains/ipv6.md b/docs/en/sql-reference/data-types/domains/ipv6.md index 7fd88887acc..bc57202bf66 100644 --- a/docs/en/sql-reference/data-types/domains/ipv6.md +++ b/docs/en/sql-reference/data-types/domains/ipv6.md @@ -31,7 +31,7 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; `IPv6` domain supports custom input as IPv6-strings: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.tech/docs/en/', '2a02:e980:1e::1'); SELECT * FROM hits; ``` diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index 4f449eea516..1468b48695b 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -701,13 +701,13 @@ arrayDifference(array) **Parameters** -- `array` – [Array](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [Array](https://clickhouse.tech/docs/en/data_types/array/). **Returned values** Returns an array of differences between adjacent elements. -Type: [UInt\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#uint-ranges), [Int\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#int-ranges), [Float\*](https://clickhouse.yandex/docs/en/data_types/float/). +Type: [UInt\*](https://clickhouse.tech/docs/en/data_types/int_uint/#uint-ranges), [Int\*](https://clickhouse.tech/docs/en/data_types/int_uint/#int-ranges), [Float\*](https://clickhouse.tech/docs/en/data_types/float/). **Example** @@ -753,7 +753,7 @@ arrayDistinct(array) **Parameters** -- `array` – [Array](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [Array](https://clickhouse.tech/docs/en/data_types/array/). **Returned values** diff --git a/docs/en/whats-new/changelog/2017.md b/docs/en/whats-new/changelog/2017.md index d819324b07a..3b48e23233f 100644 --- a/docs/en/whats-new/changelog/2017.md +++ b/docs/en/whats-new/changelog/2017.md @@ -24,7 +24,7 @@ This release contains bug fixes for the previous release 1.1.54310: #### New Features: {#new-features} - Custom partitioning key for the MergeTree family of table engines. -- [Kafka](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/) table engine. +- [Kafka](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) table engine. - Added support for loading [CatBoost](https://catboost.yandex/) models and applying them to data stored in ClickHouse. - Added support for time zones with non-integer offsets from UTC. - Added support for arithmetic operations with time intervals. diff --git a/docs/es/introduction/adopters.md b/docs/es/introduction/adopters.md index e41e8005cc7..4c0aa78d57b 100644 --- a/docs/es/introduction/adopters.md +++ b/docs/es/introduction/adopters.md @@ -37,7 +37,7 @@ toc_title: Adoptante | Exness | Comercio | Métricas, Registro | — | — | [Charla en ruso, mayo 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | Sistema abierto. | Red Ad | Producto principal | — | — | [Publicación de blog en japonés, julio 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | | HUYA | Video Streaming | Analítica | — | — | [Diapositivas en chino, octubre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| Idealista | Inmobiliario | Analítica | — | — | [Blog Post en Inglés, Abril 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| Idealista | Inmobiliario | Analítica | — | — | [Blog Post en Inglés, Abril 2019](https://clickhouse.tech/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | | Infovista | Red | Analítica | — | — | [Diapositivas en español, octubre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | | InnoGames | Juego | Métricas, Registro | — | — | [Diapositivas en ruso, septiembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | | Integros | Plataforma para servicios de video | Analítica | — | — | [Diapositivas en ruso, mayo 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | diff --git a/docs/es/sql-reference/data-types/domains/ipv4.md b/docs/es/sql-reference/data-types/domains/ipv4.md index c97229610d3..6e271f10fd2 100644 --- a/docs/es/sql-reference/data-types/domains/ipv4.md +++ b/docs/es/sql-reference/data-types/domains/ipv4.md @@ -33,7 +33,7 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from; `IPv4` domain admite formato de entrada personalizado como cadenas IPv4: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.tech/docs/en/', '116.106.34.242'); SELECT * FROM hits; ``` diff --git a/docs/es/sql-reference/data-types/domains/ipv6.md b/docs/es/sql-reference/data-types/domains/ipv6.md index bee82ff2898..2f45a353053 100644 --- a/docs/es/sql-reference/data-types/domains/ipv6.md +++ b/docs/es/sql-reference/data-types/domains/ipv6.md @@ -33,7 +33,7 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; `IPv6` domain admite entradas personalizadas como cadenas IPv6: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.tech/docs/en/', '2a02:e980:1e::1'); SELECT * FROM hits; ``` diff --git a/docs/es/sql-reference/functions/array-functions.md b/docs/es/sql-reference/functions/array-functions.md index 3a0ad14b24e..677996efabd 100644 --- a/docs/es/sql-reference/functions/array-functions.md +++ b/docs/es/sql-reference/functions/array-functions.md @@ -702,13 +702,13 @@ arrayDifference(array) **Parámetros** -- `array` – [Matriz](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [Matriz](https://clickhouse.tech/docs/en/data_types/array/). **Valores devueltos** Devuelve una matriz de diferencias entre los elementos adyacentes. -Tipo: [UInt\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#uint-ranges), [En\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#int-ranges), [Flotante\*](https://clickhouse.yandex/docs/en/data_types/float/). +Tipo: [UInt\*](https://clickhouse.tech/docs/en/data_types/int_uint/#uint-ranges), [En\*](https://clickhouse.tech/docs/en/data_types/int_uint/#int-ranges), [Flotante\*](https://clickhouse.tech/docs/en/data_types/float/). **Ejemplo** @@ -754,7 +754,7 @@ arrayDistinct(array) **Parámetros** -- `array` – [Matriz](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [Matriz](https://clickhouse.tech/docs/en/data_types/array/). **Valores devueltos** diff --git a/docs/es/whats-new/changelog/2017.md b/docs/es/whats-new/changelog/2017.md index 97b2cafd198..33e48b0409f 100644 --- a/docs/es/whats-new/changelog/2017.md +++ b/docs/es/whats-new/changelog/2017.md @@ -26,7 +26,7 @@ Esta versión contiene correcciones de errores para la versión anterior 1.1.543 #### Novedad: {#new-features} - Clave de partición personalizada para la familia MergeTree de motores de tabla. -- [Kafka](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/) motor de mesa. +- [Kafka](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) motor de mesa. - Se agregó soporte para cargar [CatBoost](https://catboost.yandex/) modelos y aplicarlos a los datos almacenados en ClickHouse. - Se agregó soporte para zonas horarias con desplazamientos no enteros de UTC. - Se agregó soporte para operaciones aritméticas con intervalos de tiempo. diff --git a/docs/fa/introduction/adopters.md b/docs/fa/introduction/adopters.md index a4ad16faf6c..654f3a24736 100644 --- a/docs/fa/introduction/adopters.md +++ b/docs/fa/introduction/adopters.md @@ -37,7 +37,7 @@ toc_title: "\u067E\u0630\u06CC\u0631\u0627" | اعمال | بازرگانی | معیارهای ورود به سیستم | — | — | [بحث در روسیه, بیشتر 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | ژنی | شبکه تبلیغاتی | محصول اصلی | — | — | [پست وبلاگ در ژاپن, جولای 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | | HUYA | جریان ویدیو | تجزیه و تحلیل | — | — | [اسلاید در چین, اکتبر 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| Idealista | املاک و مستغلات | تجزیه و تحلیل | — | — | [پست وبلاگ به زبان انگلیسی, مارس 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| Idealista | املاک و مستغلات | تجزیه و تحلیل | — | — | [پست وبلاگ به زبان انگلیسی, مارس 2019](https://clickhouse.tech/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | | اینفویستا | شبکه ها | تجزیه و تحلیل | — | — | [اسلاید به زبان انگلیسی, اکتبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | | نام | بازی ها | معیارهای ورود به سیستم | — | — | [اسلاید در روسیه, سپتامبر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | | پوششی | بستر های نرم افزاری برای خدمات تصویری | تجزیه و تحلیل | — | — | [اسلاید در روسیه, بیشتر 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | diff --git a/docs/fa/sql-reference/data-types/domains/ipv4.md b/docs/fa/sql-reference/data-types/domains/ipv4.md index 645e839f6d8..a010409d58b 100644 --- a/docs/fa/sql-reference/data-types/domains/ipv4.md +++ b/docs/fa/sql-reference/data-types/domains/ipv4.md @@ -33,7 +33,7 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from; `IPv4` دامنه پشتیبانی از فرمت ورودی سفارشی به عنوان ایپو4 رشته: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.tech/docs/en/', '116.106.34.242'); SELECT * FROM hits; ``` diff --git a/docs/fa/sql-reference/data-types/domains/ipv6.md b/docs/fa/sql-reference/data-types/domains/ipv6.md index 6677916c49b..64a9487cb07 100644 --- a/docs/fa/sql-reference/data-types/domains/ipv6.md +++ b/docs/fa/sql-reference/data-types/domains/ipv6.md @@ -33,7 +33,7 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; `IPv6` دامنه پشتیبانی از ورودی های سفارشی به عنوان ایپو6 رشته: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.tech/docs/en/', '2a02:e980:1e::1'); SELECT * FROM hits; ``` diff --git a/docs/fa/sql-reference/functions/array-functions.md b/docs/fa/sql-reference/functions/array-functions.md index 1988ed4266e..6f4e8326557 100644 --- a/docs/fa/sql-reference/functions/array-functions.md +++ b/docs/fa/sql-reference/functions/array-functions.md @@ -702,13 +702,13 @@ arrayDifference(array) **پارامترها** -- `array` – [& حذف](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [& حذف](https://clickhouse.tech/docs/en/data_types/array/). **مقادیر بازگشتی** بازگرداندن مجموعه ای از تفاوت بین عناصر مجاور. -نوع: [اینترنت\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#uint-ranges), [Int\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#int-ranges), [شناور\*](https://clickhouse.yandex/docs/en/data_types/float/). +نوع: [اینترنت\*](https://clickhouse.tech/docs/en/data_types/int_uint/#uint-ranges), [Int\*](https://clickhouse.tech/docs/en/data_types/int_uint/#int-ranges), [شناور\*](https://clickhouse.tech/docs/en/data_types/float/). **مثال** @@ -754,7 +754,7 @@ arrayDistinct(array) **پارامترها** -- `array` – [& حذف](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [& حذف](https://clickhouse.tech/docs/en/data_types/array/). **مقادیر بازگشتی** diff --git a/docs/fa/whats-new/changelog/2017.md b/docs/fa/whats-new/changelog/2017.md index 939ed966c22..ea4946cf185 100644 --- a/docs/fa/whats-new/changelog/2017.md +++ b/docs/fa/whats-new/changelog/2017.md @@ -26,7 +26,7 @@ toc_title: '2017' #### ویژگی های جدید: {#new-features} - کلید پارتیشن بندی سفارشی برای خانواده ادغام موتورهای جدول. -- [کافکا](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/) موتور جدول. +- [کافکا](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) موتور جدول. - اضافه شدن پشتیبانی برای بارگذاری [مانتو](https://catboost.yandex/) مدل ها و استفاده از داده های ذخیره شده در کلیک. - اضافه شدن پشتیبانی برای مناطق زمانی با شیپور خاموشی غیر عدد صحیح از مجموعه مقالات. - اضافه شدن پشتیبانی برای عملیات ریاضی با فواصل زمانی. diff --git a/docs/fr/introduction/adopters.md b/docs/fr/introduction/adopters.md index 833fc111fbe..e970c61955c 100644 --- a/docs/fr/introduction/adopters.md +++ b/docs/fr/introduction/adopters.md @@ -37,7 +37,7 @@ toc_title: Adoptant | Exness | Trading | Métriques, Journalisation | — | — | [Parler en russe, mai 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | Geniee | Réseau publicitaire | Produit principal | — | — | [Billet de Blog en japonais, juillet 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | | HUYA | Le Streaming Vidéo | Analytics | — | — | [Diapositives en chinois, octobre 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| Idealista | Immobilier | Analytics | — | — | [Billet de Blog en anglais, avril 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| Idealista | Immobilier | Analytics | — | — | [Billet de Blog en anglais, avril 2019](https://clickhouse.tech/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | | Infovista | Réseau | Analytics | — | — | [Diapositives en anglais, octobre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | | InnoGames | Jeu | Métriques, Journalisation | — | — | [Diapositives en russe, septembre 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | | Integros | Plate-forme pour les services vidéo | Analytics | — | — | [Diapositives en russe, mai 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | diff --git a/docs/fr/sql-reference/data-types/domains/ipv4.md b/docs/fr/sql-reference/data-types/domains/ipv4.md index 7cf36c0aaef..12895992e77 100644 --- a/docs/fr/sql-reference/data-types/domains/ipv4.md +++ b/docs/fr/sql-reference/data-types/domains/ipv4.md @@ -33,7 +33,7 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from; `IPv4` le domaine prend en charge le format d'entrée personnalisé en tant que chaînes IPv4: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.tech/docs/en/', '116.106.34.242'); SELECT * FROM hits; ``` diff --git a/docs/fr/sql-reference/data-types/domains/ipv6.md b/docs/fr/sql-reference/data-types/domains/ipv6.md index 1d0f3cd47fd..77510a950cb 100644 --- a/docs/fr/sql-reference/data-types/domains/ipv6.md +++ b/docs/fr/sql-reference/data-types/domains/ipv6.md @@ -33,7 +33,7 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; `IPv6` le domaine prend en charge l'entrée personnalisée en tant que chaînes IPv6: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.tech/docs/en/', '2a02:e980:1e::1'); SELECT * FROM hits; ``` diff --git a/docs/fr/sql-reference/functions/array-functions.md b/docs/fr/sql-reference/functions/array-functions.md index 5590774732d..ef09800614f 100644 --- a/docs/fr/sql-reference/functions/array-functions.md +++ b/docs/fr/sql-reference/functions/array-functions.md @@ -702,13 +702,13 @@ arrayDifference(array) **Paramètre** -- `array` – [Tableau](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [Tableau](https://clickhouse.tech/docs/en/data_types/array/). **Valeurs renvoyées** Renvoie un tableau de différences entre les éléments adjacents. -Type: [UInt\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#uint-ranges), [Int\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#int-ranges), [Flottant\*](https://clickhouse.yandex/docs/en/data_types/float/). +Type: [UInt\*](https://clickhouse.tech/docs/en/data_types/int_uint/#uint-ranges), [Int\*](https://clickhouse.tech/docs/en/data_types/int_uint/#int-ranges), [Flottant\*](https://clickhouse.tech/docs/en/data_types/float/). **Exemple** @@ -754,7 +754,7 @@ arrayDistinct(array) **Paramètre** -- `array` – [Tableau](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [Tableau](https://clickhouse.tech/docs/en/data_types/array/). **Valeurs renvoyées** diff --git a/docs/fr/whats-new/changelog/2017.md b/docs/fr/whats-new/changelog/2017.md index be2cb7de9f4..c812f345fdd 100644 --- a/docs/fr/whats-new/changelog/2017.md +++ b/docs/fr/whats-new/changelog/2017.md @@ -26,7 +26,7 @@ Cette version contient des corrections de bugs pour la version précédente 1.1. #### Nouveauté: {#new-features} - Clé de partitionnement personnalisée pour la famille MergeTree des moteurs de table. -- [Kafka](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/) tableau moteur. +- [Kafka](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) tableau moteur. - Ajout du support pour le chargement [CatBoost](https://catboost.yandex/) modèles et les appliquer aux données stockées dans ClickHouse. - Ajout du support pour les fuseaux horaires avec des décalages non entiers de UTC. - Ajout du support pour les opérations arithmétiques avec des intervalles de temps. diff --git a/docs/ja/introduction/adopters.md b/docs/ja/introduction/adopters.md index 084b5034a62..a1a89f6795f 100644 --- a/docs/ja/introduction/adopters.md +++ b/docs/ja/introduction/adopters.md @@ -37,7 +37,7 @@ toc_title: "\u30A2\u30C0\u30D7\u30BF\u30FC" | Exness | 取引 | 指標、ロギング | — | — | [ロシア語で話す,May2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | 魔神 | 広告ネットワーク | 主な製品 | — | — | [ブログ投稿日本語,July2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | | HUYA | ビデオストリーミング | 分析 | — | — | [中国語でのスライド,October2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| イデアリスタ | 不動産 | 分析 | — | — | [ブログ投稿英語,April2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| イデアリスタ | 不動産 | 分析 | — | — | [ブログ投稿英語,April2019](https://clickhouse.tech/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | | インフォビスタ | ネット | 分析 | — | — | [2019年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | | InnoGames | ゲーム | 指標、ロギング | — | — | [2019年ロシア](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | | インテグロス | Platformビデオサービス | 分析 | — | — | [ロシア語でのスライド,月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | diff --git a/docs/ja/introduction/distinctive-features.md b/docs/ja/introduction/distinctive-features.md index 5cf44ee0002..88dc91e0a3b 100644 --- a/docs/ja/introduction/distinctive-features.md +++ b/docs/ja/introduction/distinctive-features.md @@ -69,4 +69,4 @@ ClickHouseには、精度を犠牲にしてパフォーマンスを得るため 2. 既に挿入されたデータの変更または削除を、高頻度かつ低遅延に行う機能はありません。 [GDPR](https://gdpr-info.eu)に準拠するなど、データをクリーンアップまたは変更するために、バッチ削除およびバッチ更新が利用可能です。 3. インデックスが疎であるため、ClickHouseは、キーで単一行を取得するようなクエリにはあまり適していません。 -[Original article](https://clickhouse.yandex/docs/en/introduction/distinctive_features/) +[Original article](https://clickhouse.tech/docs/en/introduction/distinctive_features/) diff --git a/docs/ja/introduction/history.md b/docs/ja/introduction/history.md index af5dc40145d..162ed3ba415 100644 --- a/docs/ja/introduction/history.md +++ b/docs/ja/introduction/history.md @@ -48,4 +48,4 @@ Yandex.Metricaには、Metrageと呼ばれるデータを集計するための OLAPServerの制限を取り除き、レポートのための非集計データを扱う問題を解決するために、私達は ClickHouse DBMSを開発しました。 -[Original article](https://clickhouse.yandex/docs/en/introduction/history/) +[Original article](https://clickhouse.tech/docs/en/introduction/history/) diff --git a/docs/ja/introduction/performance.md b/docs/ja/introduction/performance.md index d6404853ccd..7750a10c0ec 100644 --- a/docs/ja/introduction/performance.md +++ b/docs/ja/introduction/performance.md @@ -5,9 +5,9 @@ toc_title: "\u30D1\u30D5\u30A9\u30FC\u30DE\u30F3\u30B9" # パフォーマンス {#pahuomansu} -Yandexの内部テスト結果によると、ClickHouseは、テスト可能なクラスのシステム間で同等の動作シナリオで最高のパフォーマンス(長時間のクエリで最も高いスループットと、短時間のクエリで最小のレイテンシの両方)を示します。 [別のページで](https://clickhouse.yandex/benchmark/dbms/)テスト結果を表示できます 。 +Yandexの内部テスト結果によると、ClickHouseは、テスト可能なクラスのシステム間で同等の動作シナリオで最高のパフォーマンス(長時間のクエリで最も高いスループットと、短時間のクエリで最小のレイテンシの両方)を示します。 [別のページで](https://clickhouse.tech/benchmark/dbms/)テスト結果を表示できます 。 -これは、多数の独立したベンチマークでも確認されています。インターネット検索で見つけることは難しくありませんし、 [私達がまとめた関連リンク集](https://clickhouse.yandex/#independent-benchmarks) から見つけることもできます。 +これは、多数の独立したベンチマークでも確認されています。インターネット検索で見つけることは難しくありませんし、 [私達がまとめた関連リンク集](https://clickhouse.tech/#independent-benchmarks) から見つけることもできます。 ## 単一の巨大なクエリのスループット {#dan-yi-noju-da-nakuerinosurupututo} @@ -27,4 +27,4 @@ Yandexの内部テスト結果によると、ClickHouseは、テスト可能な 少なくとも1000行のパケットにデータを挿入することをお勧めします。または、1秒あたり1回のリクエストを超えないでください。タブ区切りのダンプデータをMergeTreeテーブルに挿入する場合、挿入速度は50〜200MB/sになります。挿入された行のサイズが約1Kbの場合、速度は毎秒50,000〜200,000行になります。行が小さい場合、パフォーマンスは1秒あたりの行数で高くなります(Banner System データ- `>` 500,000行/秒、Graphite データ- `>` 1,000,000行/秒)。パフォーマンスを向上させるために、複数のINSERTクエリを並行して作成することで、パフォーマンスを線形に向上できます。 -[Original article](https://clickhouse.yandex/docs/ja/introduction/performance/) +[Original article](https://clickhouse.tech/docs/ja/introduction/performance/) diff --git a/docs/ja/sql-reference/data-types/domains/ipv4.md b/docs/ja/sql-reference/data-types/domains/ipv4.md index e355ae4f70f..c329028ad40 100644 --- a/docs/ja/sql-reference/data-types/domains/ipv4.md +++ b/docs/ja/sql-reference/data-types/domains/ipv4.md @@ -33,7 +33,7 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from; `IPv4` ドメインはIPv4文字列としてカスタム入力形式をサポート: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.tech/docs/en/', '116.106.34.242'); SELECT * FROM hits; ``` diff --git a/docs/ja/sql-reference/data-types/domains/ipv6.md b/docs/ja/sql-reference/data-types/domains/ipv6.md index 73227e7a2b7..26583429ec8 100644 --- a/docs/ja/sql-reference/data-types/domains/ipv6.md +++ b/docs/ja/sql-reference/data-types/domains/ipv6.md @@ -33,7 +33,7 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; `IPv6` ドメイ: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.tech/docs/en/', '2a02:e980:1e::1'); SELECT * FROM hits; ``` diff --git a/docs/ja/sql-reference/functions/array-functions.md b/docs/ja/sql-reference/functions/array-functions.md index 5a70770a54b..bd30262cc1e 100644 --- a/docs/ja/sql-reference/functions/array-functions.md +++ b/docs/ja/sql-reference/functions/array-functions.md @@ -702,13 +702,13 @@ arrayDifference(array) **パラメータ** -- `array` – [配列](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [配列](https://clickhouse.tech/docs/en/data_types/array/). **戻り値** 隣接する要素間の差分の配列を返します。 -タイプ: [UInt\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#uint-ranges), [Int\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#int-ranges), [フロート\*](https://clickhouse.yandex/docs/en/data_types/float/). +タイプ: [UInt\*](https://clickhouse.tech/docs/en/data_types/int_uint/#uint-ranges), [Int\*](https://clickhouse.tech/docs/en/data_types/int_uint/#int-ranges), [フロート\*](https://clickhouse.tech/docs/en/data_types/float/). **例** @@ -754,7 +754,7 @@ arrayDistinct(array) **パラメータ** -- `array` – [配列](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [配列](https://clickhouse.tech/docs/en/data_types/array/). **戻り値** diff --git a/docs/ja/whats-new/changelog/2017.md b/docs/ja/whats-new/changelog/2017.md index ada7b74e431..9561062f31d 100644 --- a/docs/ja/whats-new/changelog/2017.md +++ b/docs/ja/whats-new/changelog/2017.md @@ -26,7 +26,7 @@ toc_title: '2017' #### 新しい機能: {#new-features} - カスタムパーティショニングキーのMergeTree家族のテーブルエンジンです。 -- [カフカ](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/) テーブルエンジン。 +- [カフカ](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) テーブルエンジン。 - ロードのサポートを追加 [CatBoost](https://catboost.yandex/) モデルとClickHouseに格納されたデータにそれらを適用します。 - サポートが追加された時間帯と非整数オフセットからのUTCです。 - 時間間隔での算術演算のサポートが追加されました。 diff --git a/docs/ru/sql-reference/data-types/domains/ipv4.md b/docs/ru/sql-reference/data-types/domains/ipv4.md index 2903404774b..68b67bcca60 100644 --- a/docs/ru/sql-reference/data-types/domains/ipv4.md +++ b/docs/ru/sql-reference/data-types/domains/ipv4.md @@ -26,7 +26,7 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from; `IPv4` поддерживает вставку в виде строк с текстовым представлением IPv4 адреса: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.tech/docs/en/', '116.106.34.242'); SELECT * FROM hits; ``` diff --git a/docs/ru/sql-reference/data-types/domains/ipv6.md b/docs/ru/sql-reference/data-types/domains/ipv6.md index 045a2ad1960..c88ee74adea 100644 --- a/docs/ru/sql-reference/data-types/domains/ipv6.md +++ b/docs/ru/sql-reference/data-types/domains/ipv6.md @@ -26,7 +26,7 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; `IPv6` поддерживает вставку в виде строк с текстовым представлением IPv6 адреса: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.tech/docs/en/', '2a02:e980:1e::1'); SELECT * FROM hits; ``` diff --git a/docs/ru/sql-reference/functions/array-functions.md b/docs/ru/sql-reference/functions/array-functions.md index 71b6bda47d0..7abebc6a059 100644 --- a/docs/ru/sql-reference/functions/array-functions.md +++ b/docs/ru/sql-reference/functions/array-functions.md @@ -692,7 +692,7 @@ arrayDifference(array) **Параметры** -- `array` – [Массив](https://clickhouse.yandex/docs/ru/data_types/array/). +- `array` – [Массив](https://clickhouse.tech/docs/ru/data_types/array/). **Возвращаемое значение** @@ -742,7 +742,7 @@ arrayDistinct(array) **Параметры** -- `array` – [Массив](https://clickhouse.yandex/docs/ru/data_types/array/). +- `array` – [Массив](https://clickhouse.tech/docs/ru/data_types/array/). **Возвращаемое значение** diff --git a/docs/tools/blog.py b/docs/tools/blog.py new file mode 100644 index 00000000000..f5415bec608 --- /dev/null +++ b/docs/tools/blog.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +import datetime +import logging +import os +import time + +import nav # monkey patches mkdocs + +import mkdocs.commands +from mkdocs import config +from mkdocs import exceptions + +import mdx_clickhouse +import redirects + +import util + + +def build_for_lang(lang, args): + logging.info(f'Building {lang} blog') + + try: + theme_cfg = { + 'name': None, + 'custom_dir': os.path.join(os.path.dirname(__file__), '..', args.theme_dir), + 'language': lang, + 'direction': 'ltr', + 'static_templates': ['404.html'], + 'extra': { + 'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching + } + } + + # the following list of languages is sorted according to + # https://en.wikipedia.org/wiki/List_of_languages_by_total_number_of_speakers + languages = { + 'en': 'English', + 'ru': 'Русский' + } + + site_names = { + 'en': 'ClickHouse Blog', + 'ru': 'Блог ClickHouse ' + } + + assert len(site_names) == len(languages) + + site_dir = os.path.join(args.blog_output_dir, lang) + + plugins = ['macros'] + if args.htmlproofer: + plugins.append('htmlproofer') + + website_url = 'https://clickhouse.tech' + site_name = site_names.get(lang, site_names['en']) + blog_nav, post_meta = nav.build_blog_nav(lang, args) + raw_config = dict( + site_name=site_name, + site_url=f'{website_url}/blog/{lang}/', + docs_dir=os.path.join(args.blog_dir, lang), + site_dir=site_dir, + strict=True, + theme=theme_cfg, + nav=blog_nav, + copyright='©2016–2020 Yandex LLC', + use_directory_urls=True, + repo_name='ClickHouse/ClickHouse', + repo_url='https://github.com/ClickHouse/ClickHouse/', + edit_uri=f'edit/master/website/blog/{lang}', + markdown_extensions=mdx_clickhouse.MARKDOWN_EXTENSIONS, + plugins=plugins, + extra=dict( + now=datetime.datetime.now().isoformat(), + rev=args.rev, + rev_short=args.rev_short, + rev_url=args.rev_url, + website_url=website_url, + events=args.events, + languages=languages, + includes_dir=os.path.join(os.path.dirname(__file__), '..', '_includes'), + is_amp=False, + is_blog=True, + post_meta=post_meta + ) + ) + + cfg = config.load_config(**raw_config) + mkdocs.commands.build.build(cfg) + + redirects.build_blog_redirects(args) + + # TODO: AMP for blog + # if not args.skip_amp: + # amp.build_amp(lang, args, cfg) + + logging.info(f'Finished building {lang} blog') + + except exceptions.ConfigurationError as e: + raise SystemExit('\n' + str(e)) + + +def build_blog(args): + tasks = [] + for lang in args.blog_lang.split(','): + if lang: + tasks.append((lang, args,)) + util.run_function_in_parallel(build_for_lang, tasks, threads=False) diff --git a/docs/tools/build.py b/docs/tools/build.py index b7ddbc29629..1c8165fb36f 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -20,8 +20,8 @@ from mkdocs import exceptions import mkdocs.commands.build import amp +import blog import mdx_clickhouse - import redirects import single_page import test @@ -95,25 +95,6 @@ def build_for_lang(lang, args): else: site_dir = os.path.join(args.docs_output_dir, lang) - markdown_extensions = [ - 'mdx_clickhouse', - 'admonition', - 'attr_list', - 'codehilite', - 'nl2br', - 'sane_lists', - 'pymdownx.details', - 'pymdownx.magiclink', - 'pymdownx.superfences', - 'extra', - { - 'toc': { - 'permalink': True, - 'slugify': mdx_clickhouse.slugify - } - } - ] - plugins = ['macros'] if args.htmlproofer: plugins.append('htmlproofer') @@ -133,7 +114,7 @@ def build_for_lang(lang, args): repo_name='ClickHouse/ClickHouse', repo_url='https://github.com/ClickHouse/ClickHouse/', edit_uri=f'edit/master/docs/{lang}', - markdown_extensions=markdown_extensions, + markdown_extensions=mdx_clickhouse.MARKDOWN_EXTENSIONS, plugins=plugins, extra=dict( now=datetime.datetime.now().isoformat(), @@ -147,14 +128,15 @@ def build_for_lang(lang, args): events=args.events, languages=languages, includes_dir=os.path.join(os.path.dirname(__file__), '..', '_includes'), - is_amp=False + is_amp=False, + is_blog=False ) ) if os.path.exists(config_path): raw_config['config_file'] = config_path else: - raw_config['nav'] = nav.build_nav(lang, args) + raw_config['nav'] = nav.build_docs_nav(lang, args) cfg = config.load_config(**raw_config) @@ -187,7 +169,7 @@ def build_docs(args): if lang: tasks.append((lang, args,)) util.run_function_in_parallel(build_for_lang, tasks, threads=False) - redirects.build_redirects(args) + redirects.build_docs_redirects(args) def build(args): @@ -204,6 +186,9 @@ def build(args): from github import build_releases build_releases(args, build_docs) + if not args.skip_blog: + blog.build_blog(args) + if not args.skip_website: website.process_benchmark_results(args) website.minify_website(args) @@ -215,9 +200,11 @@ if __name__ == '__main__': website_dir = os.path.join('..', 'website') arg_parser = argparse.ArgumentParser() arg_parser.add_argument('--lang', default='en,es,fr,ru,zh,ja,tr,fa') + arg_parser.add_argument('--blog-lang', default='en,ru') arg_parser.add_argument('--docs-dir', default='.') arg_parser.add_argument('--theme-dir', default=website_dir) arg_parser.add_argument('--website-dir', default=website_dir) + arg_parser.add_argument('--blog-dir', default=os.path.join(website_dir, 'blog')) arg_parser.add_argument('--output-dir', default='build') arg_parser.add_argument('--enable-stable-releases', action='store_true') arg_parser.add_argument('--stable-releases-limit', type=int, default='3') @@ -230,6 +217,7 @@ if __name__ == '__main__': arg_parser.add_argument('--skip-amp', action='store_true') arg_parser.add_argument('--skip-pdf', action='store_true') arg_parser.add_argument('--skip-website', action='store_true') + arg_parser.add_argument('--skip-blog', action='store_true') arg_parser.add_argument('--skip-git-log', action='store_true') arg_parser.add_argument('--test-only', action='store_true') arg_parser.add_argument('--minify', action='store_true') @@ -249,6 +237,7 @@ if __name__ == '__main__': logging.getLogger('MARKDOWN').setLevel(logging.INFO) args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), 'docs') + args.blog_output_dir = os.path.join(os.path.abspath(args.output_dir), 'blog') from github import choose_latest_releases, get_events args.stable_releases = choose_latest_releases(args) if args.enable_stable_releases else [] @@ -259,6 +248,7 @@ if __name__ == '__main__': if args.test_only: args.skip_multi_page = True + args.skip_blog = True args.skip_website = True args.skip_pdf = True args.skip_amp = True diff --git a/docs/tools/mdx_clickhouse.py b/docs/tools/mdx_clickhouse.py index 393658be2d7..5ea93002cd2 100755 --- a/docs/tools/mdx_clickhouse.py +++ b/docs/tools/mdx_clickhouse.py @@ -18,6 +18,30 @@ import amp import website +def slugify(value, separator): + return slugify_impl.slugify(value, separator=separator, word_boundary=True, save_order=True) + + +MARKDOWN_EXTENSIONS = [ + 'mdx_clickhouse', + 'admonition', + 'attr_list', + 'codehilite', + 'nl2br', + 'sane_lists', + 'pymdownx.details', + 'pymdownx.magiclink', + 'pymdownx.superfences', + 'extra', + { + 'toc': { + 'permalink': True, + 'slugify': slugify + } + } +] + + class ClickHouseLinkMixin(object): def handleMatch(self, m, data): @@ -72,10 +96,6 @@ def makeExtension(**kwargs): return ClickHouseMarkdown(**kwargs) -def slugify(value, separator): - return slugify_impl.slugify(value, separator=separator, word_boundary=True, save_order=True) - - def get_translations(dirname, lang): import babel.support return babel.support.Translations.load( diff --git a/docs/tools/nav.py b/docs/tools/nav.py index 3c4fd304bd3..71bd2d8052f 100644 --- a/docs/tools/nav.py +++ b/docs/tools/nav.py @@ -1,4 +1,5 @@ import collections +import datetime import logging import os @@ -19,7 +20,8 @@ def build_nav_entry(root, args): return None, None, None result_items = [] index_meta, index_content = util.read_md_file(os.path.join(root, 'index.md')) - current_title = index_meta.get('toc_folder_title', index_meta.get('toc_title', find_first_header(index_content))) + current_title = index_meta.get('toc_folder_title', index_meta.get('toc_title')) + current_title = current_title or index_meta.get('title', find_first_header(index_content)) for filename in os.listdir(root): path = os.path.join(root, filename) if os.path.isdir(path): @@ -47,7 +49,7 @@ def build_nav_entry(root, args): return index_meta.get('toc_priority', 10000), current_title, result -def build_nav(lang, args): +def build_docs_nav(lang, args): docs_dir = os.path.join(args.docs_dir, lang) _, _, nav = build_nav_entry(docs_dir, args) result = [] @@ -64,10 +66,50 @@ def build_nav(lang, args): key = list(result[0].keys())[0] result[0][key][index_key] = 'index.md' result[0][key].move_to_end(index_key, last=False) - print('result', result) return result +def build_blog_nav(lang, args): + blog_dir = os.path.join(args.blog_dir, lang) + years = sorted(os.listdir(blog_dir), reverse=True) + result_nav = [{'hidden': 'index.md'}] + post_meta = collections.OrderedDict() + for year in years: + year_dir = os.path.join(blog_dir, year) + if not os.path.isdir(year_dir): + continue + result_nav.append({year: collections.OrderedDict()}) + posts = [] + post_meta_items = [] + for post in os.listdir(year_dir): + meta, _ = util.read_md_file(os.path.join(year_dir, post)) + post_date = meta['date'] + post_title = meta['title'] + if datetime.date.fromisoformat(post_date) > datetime.date.today(): + continue + posts.append( + (post_date, post_title, os.path.join(year, post),) + ) + if post_title in post_meta: + raise RuntimeError(f'Duplicate post title: {post_title}') + if not post_date.startswith(f'{year}-'): + raise RuntimeError(f'Post date {post_date} doesn\'t match the folder year {year}: {post_title}') + post_url_part = post.replace('.md', '') + post_meta_items.append((post_date, { + 'date': post_date, + 'title': post_title, + 'image': meta.get('image'), + 'url': f'/blog/{lang}/{year}/{post_url_part}/' + },)) + for _, title, path in sorted(posts, reverse=True): + result_nav[-1][year][title] = path + for _, post_meta_item in sorted(post_meta_items, + reverse=True, + key=lambda item: item[0]): + post_meta[post_meta_item['title']] = post_meta_item + return result_nav, post_meta + + def _custom_get_navigation(files, config): nav_config = config['nav'] or mkdocs.structure.nav.nest_paths(f.src_path for f in files.documentation_pages()) items = mkdocs.structure.nav._data_to_navigation(nav_config, files, config) diff --git a/docs/tools/redirects.py b/docs/tools/redirects.py index fc4d60aaf5a..2f5ebc8a620 100644 --- a/docs/tools/redirects.py +++ b/docs/tools/redirects.py @@ -25,24 +25,34 @@ def write_redirect_html(out_path, to_url): ''') -def build_redirect_html(args, from_path, to_path): - for lang in args.lang.split(','): - out_path = os.path.join( - args.docs_output_dir, lang, - from_path.replace('/index.md', '/index.html').replace('.md', '/index.html') - ) - version_prefix = f'/{args.version_prefix}/' if args.version_prefix else '/' - target_path = to_path.replace('/index.md', '/').replace('.md', '/') - to_url = f'/docs{version_prefix}{lang}/{target_path}' - to_url = to_url.strip() - write_redirect_html(out_path, to_url) +def build_redirect_html(args, base_prefix, lang, output_dir, from_path, to_path): + out_path = os.path.join( + output_dir, lang, + from_path.replace('/index.md', '/index.html').replace('.md', '/index.html') + ) + version_prefix = f'/{args.version_prefix}/' if args.version_prefix else '/' + target_path = to_path.replace('/index.md', '/').replace('.md', '/') + to_url = f'/{base_prefix}{version_prefix}{lang}/{target_path}' + to_url = to_url.strip() + write_redirect_html(out_path, to_url) -def build_redirects(args): +def build_docs_redirects(args): with open(os.path.join(args.docs_dir, 'redirects.txt'), 'r') as f: for line in f: - from_path, to_path = line.split(' ', 1) - build_redirect_html(args, from_path, to_path) + for lang in args.lang.split(','): + from_path, to_path = line.split(' ', 1) + build_redirect_html(args, 'docs', lang, args.docs_output_dir, from_path, to_path) + + +def build_blog_redirects(args): + for lang in args.blog_lang.split(','): + redirects_path = os.path.join(args.blog_dir, lang, 'redirects.txt') + if os.path.exists(redirects_path): + with open(redirects_path, 'r') as f: + for line in f: + from_path, to_path = line.split(' ', 1) + build_redirect_html(args, 'blog', lang, args.blog_output_dir, from_path, to_path) def build_static_redirects(args): diff --git a/docs/tools/website.py b/docs/tools/website.py index ed950bd06e3..6d4803158a4 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -17,20 +17,56 @@ import jsmin import mdx_clickhouse +def handle_iframe(iframe, soup): + if not iframe.attrs['src'].startswith('https://www.youtube.com/'): + raise RuntimeError('iframes are allowed only for YouTube') + wrapper = soup.new_tag('div') + wrapper.attrs['class'] = ['embed-responsive', 'embed-responsive-16by9'] + iframe.insert_before(wrapper) + iframe.extract() + wrapper.insert(0, iframe) + if 'width' in iframe.attrs: + del iframe.attrs['width'] + if 'height' in iframe.attrs: + del iframe.attrs['height'] + iframe.attrs['allow'] = 'accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture' + iframe.attrs['class'] = 'embed-responsive-item' + iframe.attrs['frameborder'] = '0' + iframe.attrs['allowfullscreen'] = '1' + + def adjust_markdown_html(content): soup = bs4.BeautifulSoup( content, features='html.parser' ) + for a in soup.find_all('a'): a_class = a.attrs.get('class') if a_class and 'headerlink' in a_class: a.string = '\xa0' + + for iframe in soup.find_all('iframe'): + handle_iframe(iframe, soup) + + for img in soup.find_all('img'): + if img.attrs.get('alt') == 'iframe': + img.name = 'iframe' + img.string = '' + handle_iframe(img, soup) + continue + img_class = img.attrs.get('class') + if img_class: + img.attrs['class'] = img_class + ['img-fluid'] + else: + img.attrs['class'] = 'img-fluid' + for details in soup.find_all('details'): for summary in details.find_all('summary'): if summary.parent != details: summary.extract() details.insert(0, summary) + for div in soup.find_all('div'): div_class = div.attrs.get('class') is_admonition = div_class and 'admonition' in div.attrs.get('class') @@ -41,10 +77,12 @@ def adjust_markdown_html(content): a.attrs['class'] = a_class + ['alert-link'] else: a.attrs['class'] = 'alert-link' + for p in div.find_all('p'): p_class = p.attrs.get('class') if is_admonition and p_class and ('admonition-title' in p_class): p.attrs['class'] = p_class + ['alert-heading', 'display-6', 'mb-2'] + if is_admonition: div.attrs['role'] = 'alert' if ('info' in div_class) or ('note' in div_class): @@ -136,6 +174,7 @@ def get_css_in(args): f"'{args.website_dir}/css/bootstrap.css'", f"'{args.website_dir}/css/docsearch.css'", f"'{args.website_dir}/css/base.css'", + f"'{args.website_dir}/css/blog.css'", f"'{args.website_dir}/css/docs.css'", f"'{args.website_dir}/css/highlight.css'" ] diff --git a/docs/tr/introduction/adopters.md b/docs/tr/introduction/adopters.md index 444902e0b96..1da65ebb903 100644 --- a/docs/tr/introduction/adopters.md +++ b/docs/tr/introduction/adopters.md @@ -37,7 +37,7 @@ toc_title: Benimseyenler | Exness | Ticaret | Metrikler, Günlük Kaydı | — | — | [Rusça konuşun, Mayıs 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | Geniee | Reklam Ağı | Ana ürün | — | — | [Japonca Blog yazısı, Temmuz 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | | HUYA | Video Akışı | Analiz | — | — | [Çince slaytlar, Ekim 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| Idealista | Emlak | Analiz | — | — | [İngilizce Blog yazısı, Nisan 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| Idealista | Emlak | Analiz | — | — | [İngilizce Blog yazısı, Nisan 2019](https://clickhouse.tech/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | | Infovista | Ağlar | Analiz | — | — | [İngilizce slaytlar, Ekim 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | | Innogames | Oyun | Metrikler, Günlük Kaydı | — | — | [Rusça slaytlar, Eylül 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | | Integros | Video hizmetleri platformu | Analiz | — | — | [Rusça slaytlar, Mayıs 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | diff --git a/docs/tr/sql-reference/data-types/domains/ipv4.md b/docs/tr/sql-reference/data-types/domains/ipv4.md index 22ca6e7240c..4caf031c0c3 100644 --- a/docs/tr/sql-reference/data-types/domains/ipv4.md +++ b/docs/tr/sql-reference/data-types/domains/ipv4.md @@ -33,7 +33,7 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from; `IPv4` etki alanı IPv4 dizeleri olarak özel giriş biçimini destekler: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.tech/docs/en/', '116.106.34.242'); SELECT * FROM hits; ``` diff --git a/docs/tr/sql-reference/data-types/domains/ipv6.md b/docs/tr/sql-reference/data-types/domains/ipv6.md index 642fe397e52..7f721cc07f6 100644 --- a/docs/tr/sql-reference/data-types/domains/ipv6.md +++ b/docs/tr/sql-reference/data-types/domains/ipv6.md @@ -33,7 +33,7 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; `IPv6` etki alanı IPv6 dizeleri olarak özel girişi destekler: ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.tech/docs/en/', '2a02:e980:1e::1'); SELECT * FROM hits; ``` diff --git a/docs/tr/sql-reference/functions/array-functions.md b/docs/tr/sql-reference/functions/array-functions.md index 9ecb255ebbe..9638481db52 100644 --- a/docs/tr/sql-reference/functions/array-functions.md +++ b/docs/tr/sql-reference/functions/array-functions.md @@ -702,13 +702,13 @@ arrayDifference(array) **Parametre** -- `array` – [Dizi](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [Dizi](https://clickhouse.tech/docs/en/data_types/array/). **Döndürülen değerler** Bitişik öğeler arasındaki farklar dizisini döndürür. -Tür: [Uİnt\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#uint-ranges), [Tamsayı\*](https://clickhouse.yandex/docs/en/data_types/int_uint/#int-ranges), [Yüzdürmek\*](https://clickhouse.yandex/docs/en/data_types/float/). +Tür: [Uİnt\*](https://clickhouse.tech/docs/en/data_types/int_uint/#uint-ranges), [Tamsayı\*](https://clickhouse.tech/docs/en/data_types/int_uint/#int-ranges), [Yüzdürmek\*](https://clickhouse.tech/docs/en/data_types/float/). **Örnek** @@ -754,7 +754,7 @@ arrayDistinct(array) **Parametre** -- `array` – [Dizi](https://clickhouse.yandex/docs/en/data_types/array/). +- `array` – [Dizi](https://clickhouse.tech/docs/en/data_types/array/). **Döndürülen değerler** diff --git a/docs/tr/whats-new/changelog/2017.md b/docs/tr/whats-new/changelog/2017.md index 98643fe449a..1011ebadb84 100644 --- a/docs/tr/whats-new/changelog/2017.md +++ b/docs/tr/whats-new/changelog/2017.md @@ -26,7 +26,7 @@ Bu sürüm önceki sürüm 1.1.54310 için hata düzeltmeleri içerir: #### Yenilik: {#new-features} - Tablo motorları MergeTree ailesi için özel bölümleme anahtarı. -- [Kafka](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/) masa motoru. +- [Kafka](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) masa motoru. - Yükleme için destek eklendi [CatBoost](https://catboost.yandex/) modelleri ve ClickHouse saklanan verilere uygulayarak. - UTC olmayan tamsayı uzaklıklar ile saat dilimleri için destek eklendi. - Zaman aralıklarıyla aritmetik işlemler için destek eklendi. diff --git a/docs/zh/introduction/adopters.md b/docs/zh/introduction/adopters.md index 895ec961751..38b9ca690e3 100644 --- a/docs/zh/introduction/adopters.md +++ b/docs/zh/introduction/adopters.md @@ -35,7 +35,7 @@ toc_title: "\u91C7\u7528\u8005" | [Exness](https://www.exness.com) | 交易 | 指标，日志记录 | — | — | [俄语交谈，2019年5月](https://youtu.be/_rpU-TvSfZ8?t=3215) | | [精灵](https://geniee.co.jp) | 广告网络 | 主要产品 | — | — | [日文博客，2017年7月](https://tech.geniee.co.jp/entry/2017/07/20/160100) | | [虎牙](https://www.huya.com/) | 视频流 | 分析 | — | — | [中文幻灯片，2018年10月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| [Idealista](https://www.idealista.com) | 房地产 | 分析 | — | — | [英文博客文章,四月2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| [Idealista](https://www.idealista.com) | 房地产 | 分析 | — | — | [英文博客文章,四月2019](https://clickhouse.tech/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | | [Infovista](https://www.infovista.com/) | 网络 | 分析 | — | — | [英文幻灯片，十月2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | | [InnoGames](https://www.innogames.com) | 游戏 | 指标，日志记录 | — | — | [俄文幻灯片，2019年9月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | | [Integros](https://integros.com) | 视频服务平台 | 分析 | — | — | [俄文幻灯片，2019年5月](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | diff --git a/docs/zh/sql-reference/data-types/domains/ipv4.md b/docs/zh/sql-reference/data-types/domains/ipv4.md index 65c066fb487..9ce12025405 100644 --- a/docs/zh/sql-reference/data-types/domains/ipv4.md +++ b/docs/zh/sql-reference/data-types/domains/ipv4.md @@ -24,7 +24,7 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from; 在写入与查询时，`IPv4`类型能够识别可读性更加友好的输入输出格式： ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.tech/docs/en/', '116.106.34.242'); SELECT * FROM hits; ``` diff --git a/docs/zh/sql-reference/data-types/domains/ipv6.md b/docs/zh/sql-reference/data-types/domains/ipv6.md index bc0f95932aa..5b1afc2cd39 100644 --- a/docs/zh/sql-reference/data-types/domains/ipv6.md +++ b/docs/zh/sql-reference/data-types/domains/ipv6.md @@ -24,7 +24,7 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from; 在写入与查询时，`IPv6`类型能够识别可读性更加友好的输入输出格式： ``` sql -INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1'); +INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.tech/docs/en/', '2a02:e980:1e::1'); SELECT * FROM hits; ``` diff --git a/docs/zh/whats-new/changelog/2017.md b/docs/zh/whats-new/changelog/2017.md index de62730b093..35d839c50c9 100644 --- a/docs/zh/whats-new/changelog/2017.md +++ b/docs/zh/whats-new/changelog/2017.md @@ -26,7 +26,7 @@ toc_title: '2017' #### 新功能: {#new-features} - MergeTree表引擎系列的自定义分区键。 -- [卡夫卡](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/) 表引擎。 +- [卡夫卡](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) 表引擎。 - 增加了对加载的支持 [CatBoost](https://catboost.yandex/) 模型并将其应用到ClickHouse中存储的数据。 - 增加了对UTC非整数偏移的时区的支持。 - 增加了对具有时间间隔的算术运算的支持。 diff --git a/website/blog/README.md b/website/blog/README.md new file mode 100644 index 00000000000..89d7ddfad57 --- /dev/null +++ b/website/blog/README.md @@ -0,0 +1,47 @@ +## Introduction + +First of all, **relevant guest posts are welcome**! Especially with success stories or demonstration of ClickHouse ecosystem projects. + +The ClickHouse blog is published alongside documentation and the rest of official website. So the posts reside in this same repository in [Markdown](https://github.com/ClickHouse/ClickHouse/tree/master/docs#markdown-cheatsheet) format. + +## How To Add a New Post? + +Basically you need to create a new Markdown file at the following location inside repository `/website/blog///.md` and then [open a pull-request](https://github.com/ClickHouse/ClickHouse/compare) with it. + +Each post needs to have a `yaml` meta-header with the following fields: + +- Required: + - `title`, main name of the article. In Title Case for English. + - `date`, publication date in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format, like `YYYY-MM-DD` (can be in future to postpone publication). +- Optional: + - `image`, URL to main post image. + - `tags`, list of post tags. + +Then after header goes post content in a normal markdown (with some optional extensions). + +The recommended place to store images is this GitHub repo: . It's folder structure matches this folder with blog posts: + +- `///main.jpg` for main post image (linked in `image` header field). +- `///whatever.jpg` for other images (`png` or `gif` are acceptable as well, if necessary). + +### Example + ```markdown +--- +title: 'ClickHouse Meetup in Beijing on June 8, 2019' +image: 'https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/main.jpg' +date: '2019-06-13' +tags: ['meetup','Beijing','China','events'] +--- + +24th ClickHouse Meetup globally and 3rd one in China took place in Beijing on Dragon Boat Festival weekend, which appeared to... + +![ClickHouse branded Beijing duck](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/9.jpg) +``` + +## How To Add a New Blog Language? + +If you want to write a guest post, you are welcome to use your native language or make multiple posts in multiple languages + + Unlike documentation, blog languages are independent, i.e. they have partially overlapping sets of posts and it's ok. Most posts are written only in one language because they are not relevant to audiences of other languages. + +At the moment it's not so straightforward to set up a new language for blog and it won't be documented for now, but you can just create a language directory with the first post as described above and we'll configure the website infrastructure to include it during/after merging the pull-request. diff --git a/website/blog/en/2016/evolution-of-data-structures-in-yandex-metrica.md b/website/blog/en/2016/evolution-of-data-structures-in-yandex-metrica.md new file mode 100644 index 00000000000..44739a24191 --- /dev/null +++ b/website/blog/en/2016/evolution-of-data-structures-in-yandex-metrica.md @@ -0,0 +1,108 @@ +--- +title: 'Evolution of Data Structures in Yandex.Metrica' +image: 'https://blog-images.clickhouse.tech/en/2016/evolution-of-data-structures-in-yandex-metrica/main.jpg' +date: '2016-12-13' +tags: ['Yandex.Metrica', 'data structures', 'LSM tree', 'columnar storage'] +--- + +[Yandex.Metrica](https://metrica.yandex.com/) takes in a stream of data representing events that took place on sites or on apps. Our task is to keep this data and present it in an analyzable form. The real challenge lies in trying to determine what form the processed results should be saved in so that they are easy to work with. During the development process, we had to completely change our approach to data storage organization several times. We started with MyISAM tables, then used LSM-trees and eventually came up with column-oriented database, ClickHouse. + +At its founding, Metrica was designed as an offshoot of Yandex.Direct, the search ads service. MySQL tables with MyISAM engine were used in Direct to store statistics and it was natural to use same approach in Metrica. Initially Yandex.Metrica for websites had more than 40 “fixed” report types (for example, the visitor geography report), several in-page analytics tools (like click maps), Webvisor (tool to study individual user actions in great detail), as well as the separate report constructor. But with time to keep up with business goals the system had to become more flexible and provide more customization opportunities for customers. Nowadays instead of using fixed reports Metrica allows to freely add new dimensions (for example, in a keyword report you can break data down further by landing page), segment and compare (between, let's say, traffic sources for all visitors vs. visitors from Moscow), change your set of metrics, etc. These features demanded a completely different approach to data storage than what we used with MyISAM, we will further discuss this transition from technical perspective. + +## MyISAM + +Most SELECT queries that fetch data for reports are made with the conditions WHERE CounterID = AND Date BETWEEN min_date AND max_date. Sometimes there is also filter by region, so it made sense to use complex primary key to turn this into primary key range is read. So table schema for Metrica looks like this: CounterID, Date, RegionID -> Visits, SumVisitTime, etc. Now we'll take a look at what happens when it comes in. + +A MyISAM table is comprised of a data file and an index file. If nothing was deleted from the table and the rows did not change in length during updating, the data file will consist of serialized rows arranged in succession in the order that they were added. The index (including the primary key) is a B-tree, where the leaves contain offsets in the data file. When we read index range data, a lot of offsets in the data file are taken from the index. Then reads are issued for this set of offsets in the data file. + +Let's look at the real-life situation when the index is in RAM (key cache in MySQL or system page cache), but the table data is not cached. Let's assume that we are using HDDs. The time it takes to read data depends on the volume of data that needs to be read and how many Seek operations need to be run. The number of Seek's is determined by the locality of data on the disk. + +Data locality illustrated: +![Data locality](https://blog-images.clickhouse.tech/en/2016/evolution-of-data-structures-in-yandex-metrica/1.jpg) + +Metrica events are received in almost the same order in which they actually took place. In this incoming stream, data from different counters is scattered completely at random. In other words, incoming data is local by time, but not local by CounterID. When writing to a MyISAM table, data from different counters is also placed quite randomly. This means that to read the data report, you will need to perform about as many random reads as there are rows that we need in the table. + +A typical 7200 rpm hard disk can perform 100 to 200 random reads per second. A RAID, if used properly, can handle the same amount multiplied by number of disks in it. One five-year-old SSD can perform 30,000 random reads per second, but we cannot afford to keep our data on SSD. So in this case, if we needed to read 10,000 rows for a report, it would take more than 10 seconds, which would be totally unacceptable. + +InnoDB is much better suited to reading primary key ranges since it uses [a clustered primary key](https://en.wikipedia.org/wiki/Database_index#Clustered) (i.e., the data is stored in an orderly manner on the primary key). But InnoDB was impossible to use due to its slow write speed. If this reminds you of [TokuDB](https://www.percona.com/software/mysql-database/percona-tokudb), then read on. + +It took a lot of tricks like periodic table sorting, complicated manual partitioning schemes, and keeping data in generations to keep Yandex.Metrica working on MyISAM. This approach also had a lot of lot of operational drawbacks, for example slow replication, consistency, unreliable recovery, etc. Nevertheless, as of 2011, we stored more than 580 billion rows in MyISAM tables. + +## Metrage and OLAPServer + +Metrage is an implementation of [LSM Tree](https://en.wikipedia.org/wiki/Log-structured_merge-tree), a fairly common data structure that works well for workloads with intensive stream of writes and mostly primary key reads, like Yandex.Metrica has. LevelDB did not exist in 2010 and TokuDB was proprietary at the time. + +![LSM Tree](https://blog-images.clickhouse.tech/en/2016/evolution-of-data-structures-in-yandex-metrica/2.jpg) + +In Metrage arbitrary data structures (fixed at compile time) can be used as “rows” in it. Every row is a key, value pair. A key is a structure with comparison operations for equality and inequality. The value is an arbitrary structure with operations to update (to add something) and merge (to aggregate or combine with another value). In short, it's a CRDT. Data is located pretty locally on the hard disk, so the primary key range reads are quick. Blocks of data are effectively compressed even with fast algorithms because of ordering (in 2010 we used QuickLZ, since 2011 - LZ4). Storing data in a systematic manner enables us to use a sparse index. + +Since reading is not performed very often (even though lot of rows are read when it does) the increase in latency due to having many chunks and decompressing the data block does not matter. Reading extra rows because of the index sparsity also does not make a difference. + +After transferring reports from MyISAM to Metrage, we immediately saw an increase in Metrica interface speed. Whereas earlier the 90% of page-title reports loaded in 26 seconds, with Metrage they loaded in 0.8 seconds (total time, including time to process all database queries and follow-up data transformations). The time it takes Metrage itself to process queries (for all reports) is as follows according to percent: average = 6 ms, 90tile = 31 ms, 99tile = 334 ms. + +We've been using Metrage for five years and it has proved to be a reliable solution. As of 2015 we stored 3.37 trillion rows in Metrage and used 39 * 2 servers for this. + +Its advantages were simplicity and effectiveness, which made it a far better choice for storing data than MyISAM. Though the system still had one huge drawback: it really only works effectively with fixed reports. Metrage aggregates data and saves aggregated data. But in order to do this, you have to list all the ways in which you want to aggregate data ahead of time. So if we do this in 40 different ways, it means that Metrica will contain 40 types of reports and no more. + +To mitigate this we had to keep for a while a separate storage for custom report wizard, called OLAPServer. It is a simple and very limited implementation of a column-oriented database. It supports only one table set in compile time — a session table. Unlike Metrage, data is not updated in real-time, but rather a few times per day. The only data type supported is fixed-length numbers of 1-8 bytes, so it wasn“t suitable for reports with other kinds of data, for example URLs. + +## ClickHouse + +Using OLAPServer, we developed an understanding of how well column-oriented DBMS's handle ad-hoc analytics tasks with non-aggregated data. If you can retrieve any report from non-aggregated data, then it begs the question of whether data even needs to be aggregated in advance, as we did with Metrage. + +![](https://blog-images.clickhouse.tech/en/2016/evolution-of-data-structures-in-yandex-metrica/3.gif) + +On the one hand, pre-aggregating data can reduce the volume of data that is used at the moment when the report page is loading. On the other hand, though, aggregated data doesn't solve everything. Here are the reasons why: + +- you need to have a list of reports that your users need ahead of time; in other words, the user can't put together a custom report +- when aggregating a lot of keys, the amount of data is not reduced and aggregation is useless; when there are a lot of reports, there are too many aggregation options (combinatorial explosion) +- when aggregating high cardinality keys (for example, URLs) the amount of data does not decrease by much (by less than half) +due to this, the amount of data may not be reduced, but actually grow during aggregation +- users won't view all the reports that we calculate for them (in other words, a lot of the calculations prove useless) +- it's difficult to maintain logical consistency when storing a large number of different aggregations + +As you can see, if nothing is aggregated and we work with non-aggregated data, then it's possible that the volume of computations will even be reduced. But only working with non-aggregated data imposes very high demands on the effectiveness of the system that executes the queries. + +So if we aggregate the data in advance, then we should do it constantly (in real time), but asynchronously with respect to user queries. We should really just aggregate the data in real time; a large portion of the report being received should consist of prepared data. + +If data is not aggregated in advance, all the work has to be done at the moment the user request it (i.e. while they wait for the report page to load). This means that many billions of rows need to be processed in response to the user's query; the quicker this can be done, the better. + +For this you need a good column-oriented DBMS. The market didn‘t have any column-oriented DBMS's that would handle internet-analytics tasks on the scale of Runet (the Russian internet) well enough and would not be prohibitively expensive to license. + +Recently, as an alternative to commercial column-oriented DBMS's, solutions for efficient ad-hoc analytics of data in distributed computing systems began appearing: Cloudera Impala, Spark SQL, Presto, and Apache Drill. Although such systems can work effectively with queries for internal analytical tasks, it is difficult to imagine them as the backend for the web interface of an analytical system accessible to external users. + +At Yandex, we developed and later opensourced our own column-oriented DBMS — ClickHouse. Let's review the basic requirements that we had in mind before we proceeded to development. + +**Ability to work with large datasets.** In current Yandex.Metrica for websites, ClickHouse is used to store all data for reports. As of November, 2016, the database is comprised of 18.3 trillion rows. It‘s made up of non-aggregated data that is used to retrieve reports in real-time. Every row in the largest table contains over 200 columns. + +**The system should scale linearly.** ClickHouse allows you to increase the size of cluster by adding new servers as needed. For example, Yandex.Metrica's main cluster has increased from 60 to 426 servers in three years. In the aim of fault tolerance, our servers are spread across different data centers. ClickHouse can use all hardware resources to process a single query. This way more than 2 terabyte can be processed per second. + +**High efficiency.** We especially pride ourselves on our database's high performance. Based on the results of internal tests, ClickHouse processes queries faster than any other system we could acquire. For example, ClickHouse works an average of 2.8-3.4 times faster than Vertica. With ClickHouse there is no one silver bullet that makes the system work so quickly. + +**Functionality should be sufficient for Web analytics tools.** The database supports the SQL language dialect, subqueries and JOINs (local and distributed). There are numerous SQL extensions: functions for web analytics, arrays and nested data structures, higher-order functions, aggregate functions for approximate calculations using sketching, etc. By working with ClickHouse, you get the convenience of a relational DBMS. + +ClickHouse was initially developed by the Yandex.Metrica team. Furthermore, we were able to make the system flexible and extensible enough that it can be successfully used for different tasks. Although the database can run on large clusters, it can be installed on one server or even on a virtual machine. There are now more than a dozen different ClickHouse applications within our company. + +ClickHouse is well equipped for creating all kinds of analytical tools. Just consider: if the system can handle the challenges of Yandex.Metrica, you can be sure that ClickHouse will cope with other tasks with a lot of performance headroom to spare. + +ClickHouse works well as a time series database; at Yandex it is commonly used as the backend for Graphite instead of Ceres/Whisper. This lets us work with more than a trillion metrics on a single server. + +ClickHouse is used by analytics for internal tasks. Based on our experience at Yandex, ClickHouse performs at about three orders of magnitude higher than traditional methods of data processing (scripts on MapReduce). But this is not a simple quantitative difference. The fact of the matter is that by having such a high calculation speed, you can afford to employ radically different methods of problem solving. + +If an analyst has to make a report and they are competent at their job, they won't just go ahead and construct one report. Rather, they will start by retrieving dozens of other reports to better understand the nature of the data and test various hypotheses. It is often useful to look at data from different angles in order to posit and check new hypotheses, even if you don't have a clear goal. + +This is only possible if the data analysis speed allows you to conduct online research. The faster queries are executed, the more hypotheses you can test. Working with ClickHouse, one even gets the sense that they are able to think faster. + +In traditional systems, data is like a dead weight, figuratively speaking. You can manipulate it, but it takes a lot of time and is inconvenient. If your data is in ClickHouse though, it is much more malleable: you can study it in different cross-sections and drill down to the individual rows of data. + +## Conclusions + +Yandex.Metrica has become the second largest web-analytics system in the world. The volume of data that Metrica takes in grew from 200 million events a day in 2009 to more than 25 billion in 2016. In order to provide users with a wide variety of options while still keeping up with the increasing workload, we've had to constantly modify our approach to data storage. + +Effective hardware utilization is very important to us. In our experience, when you have a large volume of data, it's better not to worry as much about how well the system scales and instead focus on how effectively each unit of resource is used: each processor core, disk and SSD, RAM, and network. After all, if your system is already using hundreds of servers, and you have to work ten times more efficiently, it is unlikely that you can just proceed to install thousands of servers, no matter how scalable your system is. + +To maximize efficiency, it's important to customize your solution to meet the needs of specific type of workload. There is no data structure that copes well with completely different scenarios. For example, it's clear that key-value databases don't work for analytical queries. The greater the load on the system, the narrower the specialization required. One should not be afraid to use completely different data structures for different tasks. + +We were able to set things up so that Yandex.Metrica's hardware was relatively inexpensive. This has allowed us to offer the service free of charge to even very large sites and mobile apps, even larger than Yanex‘s own, while competitors typically start asking for a paid subscription plan. + + diff --git a/website/blog/en/2016/how-to-update-data-in-clickhouse.md b/website/blog/en/2016/how-to-update-data-in-clickhouse.md new file mode 100644 index 00000000000..22c2fa3ccc1 --- /dev/null +++ b/website/blog/en/2016/how-to-update-data-in-clickhouse.md @@ -0,0 +1,169 @@ +--- +title: 'How to Update Data in ClickHouse' +date: '2016-11-20' +image: 'https://blog-images.clickhouse.tech/en/2016/how-to-update-data-in-clickhouse/main.jpg' +tags: ['features', 'update', 'delete', 'CollapsingMergeTree', 'partitions'] +--- + +There is no UPDATE or DELETE commands in ClickHouse at the moment. And that's not because we have some religious believes. ClickHouse is performance-oriented system; and data modifications are hard to store and process optimally in terms of performance. + +But sometimes we have to modify data. And sometimes data should be updated in realtime. Don't worry, we have these cases covered. + +## Work with Partitions + +Data in MergeTree engine family is partitioned by partition_key engine parameter. MergeTree split all the data by this partition key. Partition size is one month. + +That's very useful in many terms. Especially when we're talking about data modification. + +## Yandex.Metrica "hits" Table + +Let's look at an example on Yandex.Metrica server mtlog02-01-1 which store some Yandex.Metrica data for year 2013. Table we are looking at contains user events we call “hits”. This is the engine description for hits table: + +``` text +ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{layer}-{shard}/hits', -- zookeeper path + '{replica}', -- settings in config describing replicas + EventDate, -- partition key column + intHash32(UserID), -- sampling key + (CounterID, EventDate, intHash32(UserID), WatchID), -- index + 8192 -- index granularity +) +``` + +You can see that the partition key column is EventDate. That means that all the data will be splitted by months using this column. + +With this SQL we can get partitions list and some stats about current partitions: + +```sql +SELECT + partition, + count() as number_of_parts, + formatReadableSize(sum(bytes)) as sum_size +FROM system.parts +WHERE + active + AND database = 'merge' + AND table = 'hits' +GROUP BY partition +ORDER BY partition; +``` +```text +┌─partition─┬─number_of_parts─┬─sum_size───┐ +│ 201306 │ 1 │ 191.34 GiB │ +│ 201307 │ 4 │ 537.86 GiB │ +│ 201308 │ 6 │ 608.77 GiB │ +│ 201309 │ 5 │ 658.68 GiB │ +│ 201310 │ 5 │ 768.74 GiB │ +│ 201311 │ 5 │ 654.61 GiB │ +└───────────┴─────────────────┴────────────┘ +``` +There are 6 partitions with a few parts in each of them. Each partition is around 600 Gb of data. Partition is strictly one piece of data for partition key, here we can see that it is months. Part is one piece of data inside partition. Basically it's one node of LSMT structure, so there are not so many of them, especially for old data. If there are too many of them, they merge and form bigger ones. + +## Partition Operations + +There is a nice set of operations to work with partitions: + +- `DETACH PARTITION` - Move a partition to the 'detached' directory and forget it. +- `DROP PARTITION` - Delete a partition. +- `ATTACH PART|PARTITION` -- Add a new part or partition from the 'detached' directory to the table. +- `FREEZE PARTITION` - Create a backup of a partition. +- `FETCH PARTITION` - Download a partition from another server. + +We can do any data management operations on partitions level: move, copy and delete. Also, special DETACH and ATTACH operations are created to simplify data manipulation. DETACH detaches partition from table, moving all data to detached directory. Data is still there and you can copy it anywhere but detached data is not visible on request level. ATTACH is the opposite: attaches data from detached directory so it become visible. + +This attach-detach commands works almost in no time so you can make your updates almost transparently to database clients. + +Here is the plan how to update data using partitions: + +- Create modified partition with updated data on another table +- Copy data for this partition to detached directory +- `DROP PARTITION` in main table +- `ATTACH PARTITION` in main table + +Partition swap especially useful for huge data updates with low frequency. But they're not so handy when you need to update a lot of data in real time. + +## Update Data on the Fly + +In Yandex.Metrica we have user sessions table. Each row is one session on a website: some pages checked, some time spent, some banners clicked. This data is updated every second: user on a website view more pages, click more buttons, and do other things. Site owner can see that actions in Yandex.Metrica interface in real time. + +So how do we do that? + +We update data not by updating that data, but adding more data about what have changed. This is usually called CRDT approach, and there is an article on Wikipedia about that. + +It was created to solve conflict problem in transactions but this concept also allows updating data. We use our own data model with this approach. We call it Incremental Log. + +## Incremental Log + +Let's look at an example. + +Here we have one session information with user identifier UserID, number of page viewed PageViews, time spent on site in seconds Duration. There is also Sign field, we describe it later. +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` +And let's say we calculate some metrics over this data. + +- `count()`- number of sessions +- `sum(PageViews)`- total number of pages all users checked +- `avg(Duration)` - average session duration, how long user usually spent on the website + +Let's say now we have update on that: user checked one more page, so we should change PageViews from 5 to 6 and Duration from 146 to 185. + +We insert two more rows: +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +First one is delete row. It's exactly the same row what we already have there but with Sign set to -1. Second one is updated row with all data set to new values. + +After that we have three rows of data: +``` text +┌──────────────UserID─┬─PageViews─┬─Duration─┬─Sign─┐ +│ 4324182021466249494 │ 5 │ 146 │ 1 │ +│ 4324182021466249494 │ 5 │ 146 │ -1 │ +│ 4324182021466249494 │ 6 │ 185 │ 1 │ +└─────────────────────┴───────────┴──────────┴──────┘ +``` + +The most important part is modified metrics calculation. We should update our queries like this: + +``` text + -- number of sessions +count() -> sum(Sign) + -- total number of pages all users checked +sum(PageViews) -> sum(Sign * PageViews) + -- average session duration, how long user usually spent on the website +avg(Duration) -> sum(Sign * Duration) / sum(Sign) +``` + +You can see that it works as expected over this data. Deleted row 'hide' old row, same values come with + and - signs inside aggregation and annihilate each other. + +Moreover, it works totally fine with changing keys for grouping. If we want to group data by PageViews, all data for PageView = 5 will be 'hidden' for this rows. + +There are some limitations with this approach: + +- It works only for metrics which can be presented through this Sign operations. It covers most cases, but it's not possible to calculate min or max values. There is an impact to uniq calculations also. But it's fine at least for Yandex.Metrica cases, and there are a lot of different analytical calculations; +- You need to remember somehow old value in external system doing updates, so you can insert this 'delete' rows; +- Some other effects; there is a [great answer](https://groups.google.com/forum/#!msg/clickhouse/VixyOUD-K68/Km8EpkCyAQAJ) on Google Groups. + +## CollapsingMergeTree + +ClickHouse has support of Incremental Log model in Collapsing engines family. + +If you use Collapsing family, 'delete' row and old 'deleted' rows will collapse during merge process. Merge is a background process of merging data into larger chunks. Here is a great article about merges and LSMT structures. + +For most cases 'delete' and 'deleted' rows will be removed in terms of days. What's important here is that you will not have any significant overhead on data size. Using Sign field on selects still required. + +Also there is FINAL modifier available over Collapsing family. Using FINAL guarantees that user will see already collapsing data, thus using Sign field isn't required. FINAL usually make tremendous performance degradation because ClickHouse have to group data by key and delete rows during SELECT execution. But it's useful when you want to check your queries or if you want to see raw, unaggregated data in their final form. + +## Future Plans + +We know that current feature set is not enough. There are some cases which do not fit to limitations. But we have huge plans, and here are some insights what we've preparing: + +- Partitions by custom key: current partitioning scheme is binded to months only. We will remove this limitation and it will be possible to create partitions by any key. All partition operations like FETCH PARTITION will be available. +- UPDATE and DELETE: there are a lot of issues with updates and deletes support. Performance degradation, consistency guarantees, distributed queries and more. But we believe that if you need to update few rows of data in your dataset, it should not be painful. It will be done. + diff --git a/website/blog/en/2016/yandex-opensources-clickhouse.md b/website/blog/en/2016/yandex-opensources-clickhouse.md new file mode 100644 index 00000000000..feffec65d79 --- /dev/null +++ b/website/blog/en/2016/yandex-opensources-clickhouse.md @@ -0,0 +1,12 @@ +--- +title: 'Yandex Opensources ClickHouse' +image: 'https://blog-images.clickhouse.tech/en/2016/yandex-opensources-clickhouse/main.jpg' +date: '2016-06-15' +tags: ['announcement', 'GitHub', 'license'] +--- + +Today [analytical DBMS ClickHouse](https://clickhouse.tech/) initially developed internally at Yandex, became available to everyone. Source code is published on [GitHub](https://github.com/ClickHouse/ClickHouse) under Apache 2.0 license. + +ClickHouse allows interactive analytical query execution on data updated in real time. System is able to scale to tens of trillions of rows and petabytes of stored data. Using ClickHouse opens up opportunities that were hard to imagine: you can store full stream of data and slice and dice it to produce reports without offline aggregation. ClickHouse was initially developed as a backend for [Yandex.Metrica](https://metrika.yandex.com/) — second largest web analytics system in the world. + +[Discussion on Hacker News](https://news.ycombinator.com/item?id=11908254). diff --git a/website/blog/en/2017/clickhouse-at-data-scale-2017.md b/website/blog/en/2017/clickhouse-at-data-scale-2017.md new file mode 100644 index 00000000000..f05cc9e1e89 --- /dev/null +++ b/website/blog/en/2017/clickhouse-at-data-scale-2017.md @@ -0,0 +1,10 @@ +--- +title: 'ClickHouse at Data@Scale 2017' +image: 'https://blog-images.clickhouse.tech/en/2017/clickhouse-at-data-scale-2017/main.jpg' +date: '2017-06-15' +tags: ['conference', 'Seattle', 'USA', 'America', 'events'] +--- + +![iframe](https://www.youtube.com/embed/bSyQahMVZ7w) + +[Slides](https://presentations.clickhouse.tech/data_at_scale/) diff --git a/website/blog/en/2017/clickhouse-at-percona-live-2017.md b/website/blog/en/2017/clickhouse-at-percona-live-2017.md new file mode 100644 index 00000000000..989dfde932d --- /dev/null +++ b/website/blog/en/2017/clickhouse-at-percona-live-2017.md @@ -0,0 +1,22 @@ +--- +title: 'ClickHouse at Percona Live 2017' +image: 'https://blog-images.clickhouse.tech/en/2017/clickhouse-at-percona-live-2017/main.jpg' +date: '2017-04-28' +tags: ['meetup', 'Santa Clara', 'Bay Area', 'California', 'USA', 'America', 'events', 'Graphouse'] +--- + +For those who haven't heard, [Percona Live](https://percona.com/live/17) is probably one of the largest international conferences about opensource database management systems, having 12 talk tracks in parallel. It's been around for many years and initially, it was focused mainly on MySQL (and had that in its name), but nowadays it is more generic and other products of this category get lots of attention too. Needless to say that for a relatively new player on the market like [ClickHouse](https://clickhouse.tech/), it's been a great opportunity to spread the word about the technology and how exactly it allows us to perform analytics on petabytes of data in real-time. + +![Percona Live](https://blog-images.clickhouse.tech/en/2017/clickhouse-at-percona-live-2017/1.jpg) + +Yandex team members had three chances to talk about ClickHouse from the stage: + +1. A large portion of [Opening Keynote](https://www.percona.com/blog/2017/04/25/percona-live-2017-day-one-keynotes/) has been dedicated to different time-series databases. ClickHouse is not really a specialized time-series database but still outperforms many alternatives if used as such. So Dmitry Andreev, Head of Yandex.Market Infrastructure Development Group, had a short talk about how ClickHouse can be used a as storage backend for Graphite using [Graphouse](https://github.com/clickhouse/graphouse), an open-source adapter that implements this. This setup is used in Yandex.Market and number of other Yandex services and have proven to be very reliable and effective. Chain of short talks has been followed by a live panel about time series in general with the same speakers including Dmitry. Unfortunately, as we figured out later, many keynote attendees perceived ClickHouse as just yet another time-series database and missed the explicitly said part that it opens up way more opportunities to analyze data. +2. Victor Tarnavsky, Head of Yandex.Metrica, and Alexey Milovidov, Head of ClickHouse Development Group, gave a full-length talk about ClickHouse overview, capabilities, features and use cases. Their video has not been recorded, but you can check out [the slides](https://presentations.clickhouse.tech/percona2017/ClickHouse%20Percona%20Santa%20Clara%202.0.pdf). +3. Later on, Dmitry Andreev went deeper on the same topic he covered on an opening keynote. He spoke in more detail about how Graphouse works, shown the benchmark results and future plans of the project. Also, [only slides](https://www.percona.com/live/17/sites/default/files/slides/clickhouse-as-timeseries-database.pdf) are available. + +![Keynote](https://blog-images.clickhouse.tech/en/2017/clickhouse-at-percona-live-2017/2.gif) + +Besides, ClickHouse has been represented in the exhibition accompanying the conference. Altinity, the private company independent from Yandex that provides consulting and support services for ClickHouse, organized the booth and invited Yandex team members to join them to talk about ClickHouse with conference attendees which appeared to be quite productive. + +![ClickHouse Booth](https://blog-images.clickhouse.tech/en/2017/clickhouse-at-percona-live-2017/3.jpg) diff --git a/website/blog/en/2017/clickhouse-meetup-in-berlin-october-5-2017.md b/website/blog/en/2017/clickhouse-meetup-in-berlin-october-5-2017.md new file mode 100644 index 00000000000..bd15350ba34 --- /dev/null +++ b/website/blog/en/2017/clickhouse-meetup-in-berlin-october-5-2017.md @@ -0,0 +1,10 @@ +--- +title: 'ClickHouse Meetup in Berlin, October 5, 2017' +image: 'https://blog-images.clickhouse.tech/en/2017/clickhouse-meetup-in-berlin-october-5-2017/main.jpg' +date: '2017-10-19' +tags: ['meetup', 'Berlin', 'Germany', 'events'] +--- + +![iframe](https://www.youtube.com/embed/videoseries?list=PL0Z2YDlm0b3hO_3kCUFZLdcIQuI3gghZ8) + +All presentations are available for download at [the event page](https://events.yandex.com/events/meetings/05-10-2017/). diff --git a/website/blog/en/2017/clickhouse-meetup-in-santa-clara-may-4-2017.md b/website/blog/en/2017/clickhouse-meetup-in-santa-clara-may-4-2017.md new file mode 100644 index 00000000000..5974a292853 --- /dev/null +++ b/website/blog/en/2017/clickhouse-meetup-in-santa-clara-may-4-2017.md @@ -0,0 +1,8 @@ +--- +title: 'ClickHouse Meetup in Santa Clara on May 4, 2017' +image: 'https://blog-images.clickhouse.tech/en/2017/clickhouse-meetup-in-santa-clara-may-4-2017/main.jpg' +date: '2017-05-11' +tags: ['meetup', 'Santa Clara', 'Bay Area', 'California', 'USA', 'America', 'events'] +--- + +After [Percona Live 2017](clickhouse-at-percona-live-2017.md), Yandex ClickHouse team stayed for one more week in San Francisco Bay Area to meet with local companies in person to talk about ClickHouse and how it can be applied to their tasks. On the last evening we even managed to organize our own meetup with active ClickHouse users in the area, not as large as we regularly host in Russia, but still had some very interesting discussions. diff --git a/website/blog/en/2017/join-the-clickhouse-meetup-in-berlin.md b/website/blog/en/2017/join-the-clickhouse-meetup-in-berlin.md new file mode 100644 index 00000000000..5521127edba --- /dev/null +++ b/website/blog/en/2017/join-the-clickhouse-meetup-in-berlin.md @@ -0,0 +1,13 @@ +--- +title: 'Join the ClickHouse Meetup in Berlin' +image: 'https://blog-images.clickhouse.tech/en/2017/join-the-clickhouse-meetup-in-berlin/main.jpg' +date: '2017-10-19' +tags: ['announcement', 'meetup', 'Berlin', 'Germany', 'events'] +--- + +Come learn about ClickHouse, our open source high-performance column-oriented database management system at a meetup on October 5, 2017 at the Park Inn at Alexanderplatz 7 in Berlin. + +ClickHouse can generate custom data reports in real time and process billions of rows and dozens of gigabytes of data per single server per second. It works up to a thousand times faster than traditional approaches. ClickHouse is linearly scalable, hardware-efficient, fault-tolerant, and can be deployed across multiple data centers. Among other features, ClickHouse offers a user-friendly SQL query dialect with a number of built-in analytics capabilities. + +Join us at the meetup to learn why hundreds of companies across Europe, US, and China are adopting ClickHouse. Through interactive talks, attendees will learn about product features, how ClickHouse can benefit them, and how to use this system in practice. +Attending the ClickHouse meetup is free. [Please register to join us](https://events.yandex.com/events/meetings/05-10-2017/). diff --git a/website/blog/en/2018/announcing-clickhouse-meetup-in-amsterdam-on-november-15.md b/website/blog/en/2018/announcing-clickhouse-meetup-in-amsterdam-on-november-15.md new file mode 100644 index 00000000000..c3534efee55 --- /dev/null +++ b/website/blog/en/2018/announcing-clickhouse-meetup-in-amsterdam-on-november-15.md @@ -0,0 +1,8 @@ +--- +title: 'Announcing ClickHouse Meetup in Amsterdam on November 15' +image: 'https://blog-images.clickhouse.tech/en/2018/announcing-clickhouse-meetup-in-amsterdam-on-november-15/main.jpg' +date: '2018-10-17' +tags: ['meetup', 'Amsterdam', 'Netherlands', 'events', 'announcement'] +--- + +Yet another meetup of ClickHouse community is planned in Europe, see detailed agenda and register on [the event page](https://events.yandex.com/events/meetings/15-11-2018/). diff --git a/website/blog/en/2018/clickhouse-at-analysys-a10-2018.md b/website/blog/en/2018/clickhouse-at-analysys-a10-2018.md new file mode 100644 index 00000000000..d3700b40e42 --- /dev/null +++ b/website/blog/en/2018/clickhouse-at-analysys-a10-2018.md @@ -0,0 +1,27 @@ +--- +title: 'ClickHouse at Analysys A10 2018' +image: 'https://blog-images.clickhouse.tech/en/2018/clickhouse-at-analysys-a10-2018/main.jpg' +date: '2018-11-04' +tags: ['conference', 'Beijing', 'China', 'events', 'Analysys', 'Asia'] +--- + +[Analysys A10](https://event.analysys.cn/pc/2018/index.html) is a large conference on Big Data that took place on October 26-27 in Beijing. Since China's population is huge, it generates a lot of data and Big Data industry is in very high demand. Yandex ClickHouse team has been honored to participate in this event alongside top management, analysts, and IT professionals from various Chinese companies. + +Each year Analysys also organizes the OLAP contest. The second year in a row the same team of Sundy Li (李本旺) and Winter Zhang (张健) wins it by using ClickHouse as the core of their solution. The task was to calculate complex marketing funnel as fast as possible. + +Sundy Li (李本旺) receives award for winning Analysys OLAP contest 2018 from William Kwok (郭炜): +![Sundy Li and William Kwok](https://blog-images.clickhouse.tech/en/2018/clickhouse-at-analysys-a10-2018/1.jpg) + +The first day of the conference we mostly spent talking with people on ClickHouse booth, while on the second day there were two technical talks about ClickHouse. + +Alexey Milovidov demonstrates ClickHouse and how it works internally: +![Alexey Milovidov](https://blog-images.clickhouse.tech/en/2018/clickhouse-at-analysys-a10-2018/2.jpg) + +Sundy Li (李本旺) explains the audience how they won the OLAP contest using ClickHouse: +![Sundy Li](https://blog-images.clickhouse.tech/en/2018/clickhouse-at-analysys-a10-2018/3.jpg) + +The next day after A10 was a dedicated ClickHouse Community Meetup in Beijing, but it deserves a separate recap post. + +Analysys A10 afterparty: +![Analysys A10 afterparty](https://blog-images.clickhouse.tech/en/2018/clickhouse-at-analysys-a10-2018/4.jpg) + diff --git a/website/blog/en/2018/clickhouse-at-percona-live-europe-2018.md b/website/blog/en/2018/clickhouse-at-percona-live-europe-2018.md new file mode 100644 index 00000000000..1e0b7e4c99d --- /dev/null +++ b/website/blog/en/2018/clickhouse-at-percona-live-europe-2018.md @@ -0,0 +1,25 @@ +--- +title: 'ClickHouse at Percona Live Europe 2018' +image: 'https://blog-images.clickhouse.tech/en/2018/clickhouse-at-percona-live-europe-2018/main.jpg' +date: '2018-11-21' +tags: ['conference', 'Frankfurt', 'Germany', 'events', 'Percona Live', 'Europe'] +--- + +Open-source database management systems conference [Percona Live](https://www.percona.com/live/e18/) this time took place on November 5-7 in Germany, Frankfurt am Main. Over last couple years ClickHouse became a solid member of this community as demand in analytics with subsecond latencies appeared to be pretty high. + +There were three talks about ClickHouse in agenda, while only one of them was from Yandex. Also we had a lot of interesting conversations with conference attendees at ClickHouse booth sponsored by Altinity. + +Alexander Zaitsev, CTO and co-founder of Altinity, gives an overview of ClickHouse and then demonstrates case studies and best practices ([slides](https://presentations.clickhouse.tech/percona_europe_2018/Altinity.pdf)): +![](https://blog-images.clickhouse.tech/en/2018/clickhouse-at-percona-live-europe-2018/1.jpg) + +Fast! Flexible! Free! Fun! +![Fast! Flexible! Free! Fun!](https://blog-images.clickhouse.tech/en/2018/clickhouse-at-percona-live-europe-2018/2.jpg) + +Aleksey Milovidov, lead ClickHouse developer from Yandex, talks about unusual and unique ClickHouse features ([slides](https://presentations.clickhouse.tech/percona_europe_2018)): +![Aleksey Milovidov](https://blog-images.clickhouse.tech/en/2018/clickhouse-at-percona-live-europe-2018/3.jpg) + +Aleksandar Aleksandrov and Felix Mattrat, data engineers from MessageBird, show how they use ClickHouse to analyze process of delivery of SMS and other kinds of messages ([slides](http://presentations.clickhouse.tech/percona_europe_2018/MessageBird.pdf)): +![Aleksandar Aleksandrov and Felix Mattrat](https://blog-images.clickhouse.tech/en/2018/clickhouse-at-percona-live-europe-2018/4.jpg) + +Live demo at ClickHouse booth by Alexey Milovidov: +![Demo at ClickHouse booth by Alexey Milovidov](https://blog-images.clickhouse.tech/en/2018/clickhouse-at-percona-live-europe-2018/5.jpg) diff --git a/website/blog/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018.md b/website/blog/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018.md new file mode 100644 index 00000000000..cd7cb155c37 --- /dev/null +++ b/website/blog/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018.md @@ -0,0 +1,68 @@ +--- +title: 'ClickHouse Community Meetup in Beijing on January 27, 2018' +image: 'https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018/main.jpg' +date: '2018-02-08' +tags: ['meetup', 'Beijing', 'China', 'events', 'Asia'] +--- + +Last year there has been an OLAP algorithm contest in China organized by Analysys. The team who have shown the top results and won the competition has been using ClickHouse as the core of their solution. Other teams were mostly using different technologies and didn't really know much about ClickHouse at a time. When the final results were published, many people in China who participated in or were aware of this competition became really eager to learn more about ClickHouse. This spike of interest about ClickHouse in China has eventually lead to the first Chinese ClickHouse Community Meetup that has taken place in Beijing. + +Welcome word by William Kwok, CTO of Analysys, who personally played a huge role in making this event possible: +![William Kwok, CTO of Analysys](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018/1.jpg) + +It was probably the most intense ClickHouse Meetup compared to all previous ones worldwide. The main part of the event took over 6 hours non-stop and there were also either pre-meetup and after-party on the same day. Well over 150 people have shown up on Saturday to participate. + +Audience listening for ClickHouse introduction by Alexey Milovidov: +![ClickHouse introduction by Alexey Milovidov](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018/2.jpg) + +Alexey Milovidov has started the main meetup session with an introductory talk about ClickHouse, it's usage inside Yandex and history that lead to becoming an open-source analytical DBMS ([slides](https://presentations.clickhouse.tech/meetup12/introduction/)). + +Alexander Zaitsev's practical talk about migrating to ClickHouse: +![Alexander Zaitsev](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018/3.jpg) + +Alexander Zaitsev has shared his vast experience in migrating to ClickHouse. LifeStreet, advertisement company where he works, was one of the first companies outside of Yandex which switched to ClickHouse from other analytical DBMS in production. Later on, Alexander also co-founded Altinity, a company that specializes in helping others to migrate to ClickHouse and then effectively use it to achieve their business goals. The talk has covered many specific topics that are important for those who are in the middle of such migration or just considering it ([Slides](https://presentations.clickhouse.tech/meetup12/migration.pptx)). + +Alexey Zatelepin explaining how ClickHouse sparse index works and other implementation details: +![Alexey Zatelepin](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018/4.jpg) + +Alexey Zatelepin's technical talk was focused on providing engineers some insights on why ClickHouse is that fast in OLAP workloads and how to leverage its design and core features as a primary index, replication, and distributed tables to achieve great performance and reliability ([slides](https://presentations.clickhouse.tech/meetup12/internals.pdf)). + +Jack Gao gives an extensive overview of ClickHouse and it's use cases in Chinese: +![Jack Gao](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018/5.jpg) + +As we have learned during meet up and the rest of our business trip, actually there are many companies in China that are already using or seriously evaluating ClickHouse to use either part of their products or for internal analytics. Three of them are doing this long and extensively enough to give a full talk about their progress and experience. + +In China, in general, and especially in Beijing the knowledge of English is not really common. Chinese people working in the IT industry have to know English well enough to read documentation, but it does not really imply that they can talk or understand verbal English well. So the talks by representatives of local companies were in Chinese. + +Jack Gao, ex-DBA and now an analyst at Sina (major social network) have dedicated a significant part of his talk to go over fundamental topics essential to most ClickHouse users. It partially overlapped with previous talks, but this time in Chinese. Also, he covered not only use case of ClickHouse in Sina but also other publicly known cases by other companies. Considering the reaction of the audience, it has been the most useful talk of the whole meetup, because of the widely useful content, lack of language barrier, and excellent execution of presentation. We even had to sacrifice initially scheduled a short break to give Jack some additional time ([slides](https://presentations.clickhouse.tech/meetup12/power_your_data.pdf)). + +Yang Xujun from Dataliance / UltraPower, which provides outsourced data analysis platform to telecom companies in China, have demonstrated why they decided to move away from reports prepared offline in Apache Hadoop / Spark and exported to MySQL towards ClickHouse. In short: Hadoop is too slow and cumbersome ([slides](https://presentations.clickhouse.tech/meetup12/telecom.pdf)). + +It might sound obvious, but the huge Chinese population generates insane amounts of data to store and process. So IT companies operating mostly on the local Chinese market are often handling amounts of information comparable to even the largest global companies. + +Kent Wang from Splunk Shanghai R&D center has demonstrated the current state of ClickHouse integration into Splunk ecosystem. Basically, they have plugged ClickHouse into their system via JDBC driver to allow data from ClickHouse to be easily accessed in Splunk UI and dashboards. Last spring Yandex ClickHouse team actually had a friendly visit to Splunk office in San Francisco to discuss potential points of interaction and exchange experience, so it was great to hear that there's some real progress in that direction ([slides](https://presentations.clickhouse.tech/meetup12/splunk.pdf)). + +The last talk was for the most tenacious ClickHouse users. Alexey Milovidov has announced some recently released features and improvements and shared what's coming next either in the short and long term [slides](https://presentations.clickhouse.tech/meetup12/news_and_plans/). + +Here is an over 5 hours long video recording of main meetup session: + +![iframe](https://www.youtube.com/embed/UXw8izZGPGk) + +If you are from China or at least can read Chinese, you might consider joining the **[Chinese ClickHouse User Group](http://www.clickhouse.com.cn/)**. + +{## Likely outdated in favor of YouTube + +There is an over 5 hours long video recording of main meetup session, but it'll take a bit of effort to get access to it (especially if you are not from China): http://m.zm518.cn/zhangmen/livenumber/share/entry/?liveId=1460023&sharerId=6fd3bac16125e71d69-899&circleId=b0b78915b2edbfe6c-78f7&followerId=×tamp=1517022274560 +You'll need to install WeChat (probably one of the most popular messengers in the world, everyone in China has it) on your smartphone: Android or iOS. https://play.google.com/store/apps/details?id=com.tencent.mm https://itunes.apple.com/ru/app/wechat/id414478124?mt=8 +On the first launch, WeChat will ask to confirm your phone number via SMS, read some digits via a microphone and accept the user agreement. Go through this. +On your computer, click the red button in the middle of the video behind the link above. It'll show a QR code. Now in WeChat in the top-right corner, there's the “+” button which opens a menu that has a “Scan QR code” item. Use it to scan QR code from your computer screen, then press the “Sign in” button on the smartphone. Now the video on the computer automatically becomes playable. +If you are from China or at least can read Chinese, you might consider joining the Chinese ClickHouse User Group. + +ClickHouse Community Meetup afterparty. +##} + +Pre-meetup meeting of speakers and most active ClickHouse users in China: +![Pre-meetup meeting](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018/6.jpg) + +ClickHouse Community Meetup afterparty: +![ClickHouse Community Meetup afterparty](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-january-27-2018/7.jpg) diff --git a/website/blog/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018.md b/website/blog/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018.md new file mode 100644 index 00000000000..a794a5f7a7e --- /dev/null +++ b/website/blog/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018.md @@ -0,0 +1,54 @@ +--- +title: 'ClickHouse Community Meetup in Beijing on October 28, 2018' +image: 'https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/main.jpg' +date: '2018-11-12' +tags: ['meetup', 'Beijing', 'China', 'Asia', 'events'] +--- + +Interest in ClickHouse among Chinese experts is growing rapidly. It was second ClickHouse Meetup in Beijing this year and the venue was more than full, it could fit only about 170 people out of 500 who signed up and around 2000 more joined the live translation online. Many Chinese companies have already adopted ClickHouse in production and are willing to share their experience. + +See the **[video recording of all talks](http://play.yunxi.tv/livestream/flash?id=05527cf6e260448b9d880b99d2cf4d40)** and **[all slides](https://github.com/yandex/clickhouse-presentations/tree/master/meetup19)**. + +Welcome word by William Kwok (郭炜), CTO of Analysys, who played a key role in organizing this event: +![William Kwok](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/1.jpg) + +Nikolay Kochetov from Yandex demonstrating recent advancements in string processing optimization using LowCardinality feature: +![Nikolay Kochetov from Yandex](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/2.jpg) + +Shang Shujie (尚书杰) from Kuaishou gives an overview of ClickHouse and it's usage scenarios: +![Shang Shujie](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/3.jpg) + +Winter Zhang (张健) from QingCloud explains their services based on ClickHouse: +![Winter Zhang](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/4.jpg) + +Audience listening to Zhang's talk: +![Audience listening to Zhang's talk](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/5.jpg) + +Li Junfei (李俊飞) from Tencent explains how ClickHouse fits their data processing infrastructure: +![Li Junfei](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/6.jpg) + +Questions&Answers session: +![Q&A](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/7.jpg) + +Jack Gao (高鹏) from Sina explains their ClickHouse use case and gives some advice based on their extensive experience with ClickHouse: +![Jack Gao](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/8.jpg) + +Chinese developers are also one of the most active worldwide in contributing to ClickHouse source code compared to other countries. Chinese ClickHouse Contributors Awards 2018 ceremony was also part of the meetup agenda with the following: + +1. 张建 (Winter Zhang, zhang2014) received First Place among independent ClickHouse developers in China for 2018, having developed 22 new features, improvements, and fixes in 57 pull requests. +2. Amos Bird received Second Place among independent ClickHouse developers in China for 2018, having developed 16 new features, improvements, and fixes in 42 pull requests. +3. 李本旺 (sundy-li) received Third Place among independent ClickHouse developers in China for 2018, having developed 6 new features, improvements, and fixes in 11 pull requests. + +A special award went to William Kwok (郭炜) for his active role in developing the Chinese ClickHouse Community. + +Sundy Li (李本旺) receives ClickHouse Contributor Award from Alexey Milovidov: +![Sundy Li](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/9.jpg) + +William Kwok (郭炜) receives special award for organizing Chinese ClickHouse community and meetups: +![William Kwok](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/10.jpg) + +Pre-meetup at the Analysys office: +![Pre-meetup](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/11.jpg) + + + diff --git a/website/blog/en/2018/clickhouse-community-meetup-in-berlin-on-july-3-2018.md b/website/blog/en/2018/clickhouse-community-meetup-in-berlin-on-july-3-2018.md new file mode 100644 index 00000000000..1db4b33c8dc --- /dev/null +++ b/website/blog/en/2018/clickhouse-community-meetup-in-berlin-on-july-3-2018.md @@ -0,0 +1,39 @@ +--- +title: 'ClickHouse Community Meetup in Berlin on July 3, 2018' +image: 'https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-berlin-on-july-3-2018/main.jpg' +date: '2018-07-05' +tags: ['meetup', 'Berlin', 'Germany', 'events'] +--- + +Just a few months ago Brenno Oliveira from Delivery Hero has dropped us an email saying that they want to host a meetup of ClickHouse community in their HQ and together we made it happen. Actually, renting a suitable room is one of the main limiting factors on how often ClickHouse meetups can happen worldwide and it was very kind of Delivery Hero to provide it for free. Bringing interesting speakers was the easy part as there are more and more companies adopting ClickHouse and willing to share their stories. Being an open-source product has its advantages after all. About 50 people have shown up from 75 sign-ups, which is way above the typical rate. + +To get started Alexander Zaitsev from Altinity gave an overview of ClickHouse for those who are not that familiar with the technology yet. He was using use cases from his personal experience and their clients as examples. Here are [the slides](https://presentations.clickhouse.tech/meetup16/introduction.pdf), unfortunately, no video this time. + +Gleb Kanterov talking about the usage of ClickHouse for experimentation metrics at Spotify: +![Gleb Kanterov Spotify](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-berlin-on-july-3-2018/1.jpg) + +![Gleb Kanterov Spotify](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-berlin-on-july-3-2018/2.jpg) + +Spotify relies heavily on what Google Cloud Platform provides, but nevertheless found a spot in their infrastructure where only ClickHouse appeared to satisfy the requirements. Gleb Kanterov has demonstrated their approach to conducting experiments and measuring if they are worth being promoted to production solutions. Using ClickHouse has allowed them to build a framework scalable to thousands of metrics, which in the end makes them move even faster and break fewer things. Checking out [full slides](https://presentations.clickhouse.tech/meetup16/spotify.pdf) is highly recommended and here are a few quotes: + +- **Requirements** + - Serve 100-s of QPS with sub-second latency + - We know in advance what are queries and data + - Maintain 10x metrics with the same cost + - Thousands of metrics + - Billions of rows per day in each of 100-s of tables + - Ready to be used out of the box + - Leverage existing infrastructure as much as feasible + - Hide unnecessary complexity from internal users +- **Why ClickHouse?** + - Build proof of concept using various OLAP storages (ClickHouse, Druid, Pinot,...) + - ClickHouse has the most simple architecture + - Powerful SQL dialect close to Standard SQL + - A comprehensive set of built-in functions and aggregators + - Was ready to be used out of the box + - Superset integration is great + - Easy to query using clickhouse-jdbc and jooq + +The last talk by Alexey Milovidov was pretty technical and mostly intended for a deeper understanding of what's going on inside ClickHouse, see [the slides](https://presentations.clickhouse.tech/meetup16/internals.pdf). There were many experienced users in the audience who didn't mind staying late to hear that and ask very relevant questions. Actually, we had to leave the building way before people were out of topics to discuss. + +If your company regularly hosts technical meetups and you are looking for interesting topics to talk about, ClickHouse might be in pretty high demand. Feel free to write Yandex ClickHouse team via [this form](http://clickhouse.tech/#meet) if you are interested to host a similar event in your city and we'll find a way to cooperate and bring in other ClickHouse community members. diff --git a/website/blog/en/2018/clickhouse-community-meetup-in-berlin-on-july-3.md b/website/blog/en/2018/clickhouse-community-meetup-in-berlin-on-july-3.md new file mode 100644 index 00000000000..7e0082cd570 --- /dev/null +++ b/website/blog/en/2018/clickhouse-community-meetup-in-berlin-on-july-3.md @@ -0,0 +1,8 @@ +--- +title: 'Announcing ClickHouse Community Meetup in Berlin on July 3' +image: 'https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-berlin-on-july-3/main.jpg' +date: '2018-06-25' +tags: ['meetup', 'Berlin', 'Germany', 'events', 'announcement'] +--- + +There's yet another upcoming meetup of ClickHouse community in Europe, see detailed agenda and sign up on [the event page](https://bitly.com/2Jv9Bug). diff --git a/website/blog/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018.md b/website/blog/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018.md new file mode 100644 index 00000000000..4f9874af655 --- /dev/null +++ b/website/blog/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018.md @@ -0,0 +1,20 @@ +--- +title: 'ClickHouse Community Meetup in Paris on October 2, 2018' +image: 'https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018/main.jpg' +date: '2018-10-09' +tags: ['meetup', 'Paris', 'France', 'events'] +--- + +Agenda of Paris ClickHouse Meetup was full of use cases, mostly from France-based companies which are actively using ClickHouse. Slides for all talks are [available on the GitHub](https://github.com/clickhouse/clickhouse-presentations/tree/master/meetup18). + +Christophe Kalenzaga and Vianney Foucault, engineers from ContentSquare, company that provided the meetup venue: +![Christophe Kalenzaga and Vianney Foucault](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018/1.jpg) + +Matthieu Jacquet from Storetail (Criteo): +![Matthieu Jacquet](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018/2.jpg) + +The audience: +![Audience](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018/3.jpg) + +Networking after the meetup: +![Networking](https://blog-images.clickhouse.tech/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018/4.jpg) diff --git a/website/blog/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018.md b/website/blog/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018.md new file mode 100644 index 00000000000..080caf610e5 --- /dev/null +++ b/website/blog/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018.md @@ -0,0 +1,27 @@ +--- +title: 'ClickHouse Meetup in Amsterdam on November 15, 2018' +image: 'https://blog-images.clickhouse.tech/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018/main.jpg' +date: '2018-11-22' +tags: ['meetup', 'Amsterdam', 'Netherlands', 'events'] +--- + +20th ClickHouse Meetup took place in Amsterdam, which appeared to be a convenient location for people from all over Europe to join the event, including Austria, Czech Republic and Germany. We were also glad to see people from many local companies including Booking.com, Crobox, Marktplaats (eBay), MessageBird and others. + +Aleksandar Aleksandrov and Felix Mattrat, data engineers from MessageBird, show how they use ClickHouse to analyze process of delivery of SMS and other kinds of messages: +![Aleksandar Aleksandrov and Felix Mattrat](https://blog-images.clickhouse.tech/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018/1.jpg) + +Nikolay Kochetov from Yandex ClickHouse team demonstrates recent features related to string processing optimization: +![Nikolay Kochetov from Yandex ClickHouse team](https://blog-images.clickhouse.tech/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018/2.jpg) + +Konstantin Ignatov from Qrator Labs shares his experience in using ClickHouse as time-series database: +![Konstantin Ignatov from Qrator Labs](https://blog-images.clickhouse.tech/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018/3.jpg) + +Aurimas Jacikevicius from Altinity demonstrates benchmark of ClickHouse against TimescaleDB and InfluxDB under time-series workload: +![Aurimas Jacikevicius from Altinity](https://blog-images.clickhouse.tech/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018/4.jpg) + +Roy Brondgeest from Crobox showcases [ClickHouse Scala reactive client](https://github.com/crobox/clickhouse-scala-client) and it's bundled [DSL for query building](https://github.com/crobox/clickhouse-scala-client/wiki): +![Roy Brondgeest from Crobox](https://blog-images.clickhouse.tech/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018/5.jpg) + +Alexey Milovidov from Yandex closes the meetup with talk about performance analysis of ClickHouse queries: +![Alexey Milovidov from Yandex](https://blog-images.clickhouse.tech/en/2018/clickhouse-meetup-in-amsterdam-on-november-15-2018/6.jpg) + diff --git a/website/blog/en/2018/concept-cloud-mergetree-tables.md b/website/blog/en/2018/concept-cloud-mergetree-tables.md new file mode 100644 index 00000000000..9d4818ba8d7 --- /dev/null +++ b/website/blog/en/2018/concept-cloud-mergetree-tables.md @@ -0,0 +1,120 @@ +--- +title: 'Concept: "Cloud" MergeTree Tables' +image: 'https://blog-images.clickhouse.tech/en/2018/concept-cloud-mergetree-tables/main.jpg' +date: '2018-11-23' +tags: ['concept', 'MergeTree', 'future', 'sharding'] +--- + +The main property of the MergeTree cloud tables is the absence of manual control over the sharding scheme of data on a cluster. The data in the cloud tables are distributed around the cluster on its own, while at the same time providing the locality property for a certain key. + +## Requirements + +1. Creating a cloud table makes it visible on all nodes of the cluster. No need to manually create a separate Distributed table and local tables on each node. +2. When ingesting data to a cloud table, while the table is very small, data is distributed across several cluster servers, but as data grows, more servers are involved (for example, starting from gigabytes per server). The user can create a small table and it should not be too cumbersome; but when creating a table, we do not know in advance how much data will be loaded into it. +3. The user specifies a sharding key (arbitrary tuple). Data for the sharding key range (in lexicographical order) is located on some servers. Very small ranges are located on several servers and to access it is enough to read data from a single server, while sufficiently large ranges are spread across all servers. For example, if we are talking about web analytics the sharding key might start with CounterID, the website identifier. Data on a large site like https://yandex.ru should be spread across all servers in the cluster, while data on a small site should be located on only a few servers. Physical explanation: the cluster should scale to simultaneously provide throughput for heavy queries and to handle high QPS of light queries, and for light queries, the latency should not suffer. In general, this is called data locality. +4. The ability for heavy queries to use all the servers in the cluster, rather than 1 / N, where N is the replication coefficient. Thus, one server can contain multiple replicas of different shards. +5. When replacing the server with an empty one (node recovery), the data restore must be parallelized in some way. At least the reads should be spread over different servers to avoid overloading individual servers. +6. On each local server, reading the range of the primary key should be touching not a very large number of file ranges or not too small file ranges (minimizing disk seeks). +7. (Optional) The ability to use individual disks instead of RAID, but at the same time preserving throughput when reading medium-sized primary key ranges and preserving QPS when reading small-sized ranges. +8. The ability to create multiple tables with a common sharding scheme (co-sharding). +9. Rebalancing data when adding new servers; creation of additional replicas with long unavailability of old servers. +10. SELECT queries should not require synchronous requests to the coordinator. No duplicates or missing data visible by SELECT queries during data rebalancing operations. +11. SELECT queries must choose large enough subset of servers considering conditions on sharding key and knowledge of the current sharding scheme. +12. The ability to efficiently distribute data across servers with uneven available disk space. +13. Atomicity of INSERT on a cluster. + +Out of scope and will not be considered: + +1. Erasure data encoding for replication and recovery. +2. Data storage on systems with different disks - HDD and SSD. An example is storing fresh data on an SSD. + +## General Considerations + +A similar problem usually (in Map-Reduce or blob-storage systems) is solved by organizing data in chunks. Chunks are located on the nodes of the cluster. Mappings: table or file -> chunks, chunk -> nodes, are stored in the master, which itself can be replicated. The master observes the liveliness of nodes and maintains a reasonable replication level of all chunks. + +Difficulties arise when there are too many chunks: in this case, the master does not cope with the storage of metadata and with the load. It becomes necessary to make complicated metadata sharding. + +In our case, it may seem tempting to solve a problem in a similar way, where instead of a chunk, an instance of a MergeTree type table containing the data range is used. Chunks in other systems are called “tablets” or “regions”. But there are many problems with this. The number of chunks on one server cannot be large, because then the property is violated - minimizing the number of seeks when reading data ranges. The problem also arises from the fact that each MergeTree table itself is rather cumbersome and consists of a large number of files. On the other hand, tables with a size of one terabyte are more or less normal if the data locality property is maintained. That is if several such tables on one server begin to be used only for not too small data ranges. + +A variety of options can be used for sharding data, including: +Sharding according to some formula with a small number of parameters. Examples are simple hashing, consistent hashing (hash ring, rendezvous hashing, jump consistent hashing, sumbur). The practice of using in other systems shows that in its pure form this approach does not work well, because the sharding scheme is poorly controlled. Fits fine, for example, for caches. It can also be used as part of another algorithm. + +The opposite option is that the data is divided into shards using an explicitly specified table. The table may contain key ranges (or, in another case, hash ranges from keys) and their corresponding servers. This gives a much greater degree of freedom in choosing when and how to transfer data. But at the same time, to scale the cluster, the size of the table has to be dynamically expanded, breaking the existing ranges. + +One of the combined options is that the mapping is made up of two parts: first, the set of various keys is divided into some pre-fixed not too few and not too many “virtual shards” (you can also call “logical shards”, “mini-shards”). This number is several times larger than the hypothetical cluster size in the number of servers. Further, the second mapping explicitly specifies the location of each mini-shard on the servers, and this second mapping can be controlled arbitrarily. + +The complexity of this approach is that partitioning hash ranges gives uniformity, but does not give locality of data for range queries; whereas when splitting by key ranges, it is difficult to choose a uniform distribution in advance since we do not know what the distribution of data will be to the keys. That is, the approach with the choice of a pre-fixed split into mini-shards does not work if data locality is required. + +It turns out that the only acceptable approach in our case is partitioning by key ranges, which can change dynamically (repartitioned). At the same time, for more convenience, manageability, and uniformity of data distribution, the number of partitioning elements can be slightly larger than the number of servers, and the mapping from the partitioning element into servers can be changed separately. + +## Possible Implementation + +Each ClickHouse server can participate in a certain cloud. The cloud is identified by a text string. The membership of a node in the cloud can be ensured by creating a certain type of database on the node (IDatabase). Thus, one node can be registered in several clouds. Registry of the nodes registered in the cloud is maintained in the coordinator. + +Cloud nodes are selected to accommodate the replicas of the shards of cloud tables. The node also sends some additional information to the coordinator for its selection when placing data: the path that determines the locality in the network (for example, data center and rack), the amount of disk space, etc. + +The cloud table is created in the corresponding database registered in the cloud. The table is created on any server and is visible in all databases registered in the cloud. + +Sharding key is set for cloud table on it“s creation, an arbitrary tuple. Sometimes it is practical that the sharding key matches the primary key (example - (CounterID, Date, UserID)), sometimes it makes sense that it is different (for example, the DateTime primary key, sharding key - UserID). + +Sharding is a composition of several mappings: + +1. The set of all possible tuples, the values ​​of the sharding key, is mapped onto many half-intervals that break the half-interval [0, 1). Initially, this number is the size of the partition, it is equal to 1. That is, all values ​​are mapped into a single semi-interval, the whole set [0, 1). Then, as the amount of data in the table increases, the semi-intervals, the split elements, can be divided approximately in half by the median of the distribution of values ​​in lexicographical order. +2. For each half-interval splitting, several cloud servers are selected and remembered in some way, on which replicas of the corresponding data will be located. The choice is made based on the location of servers on the network (for example, at least two replicas in different data centers and all replicas in different racks), the number of replicas already created on this server (choose servers with the minimum) and the amount of free space (from various servers just select the server with the maximum amount of free space). + +As a result, this composition forms a mapping from the sharding key into several replica servers. + +It is assumed that in the course of work both parts of this mapping may change. + +The result of mapping 1 can be called the “virtual shard” or “logical shard”. In the process of work, virtual shards can be divided in half. Going in the opposite direction is impossible - the number of virtual shards can only grow. It is assumed that even for tables occupying the entire cluster, the number of virtual shards will be several times larger than the number of servers (for example, it may be greater by 10 times the replication ratio). Data ranges occupying at least a tenth of all data should be spread across all servers to ensure throughput of heavy queries. The mapping as a whole is specified by the set of boundary values ​​for the sharding key. This set is small (roughly kilobytes) and stored in the coordinator. + +The mapping of virtual shards on real servers can change arbitrarily: the number of replicas can increase when servers are not available for a long time or increase and then decrease to move replicas between servers. +## How to Satisfy All Requirements + +List items below correspond to the requirement numbers above: + +1. IDatabase synchronously goes to the coordinator to get or change the list of tables. The list of cloud tables is stored in the coordinator in the node corresponding to the cloud. That is, all the tables in the cloud are visible on each server entering the cloud. +2. It is ensured by the fact that initially the partition consists of a single element, but begins to break up further with increasing data volume. Each replica responsible for the local storage of this data can initiate the splitting, once the criterion for the data volume has been reached. Multiple replicas may decide to do this competitively, and the decision is made using atomic CAS. To have fewer problems, it is possible to randomize somewhat the moment of deciding repartition. The criterion when it is necessary to additionally break virtual shards turns out to be non-trivial. For example, you can break up to the number of servers * the replication rate quite soon, by growing a shard to several gigabytes. But it is already worth breaking shards even when shards are 1 / N in size from the server size (for example, around a terabyte). In coordinator, you should store the last and previous splits immediately and do not do the splitting too often. +3. It is ensured by the fact that the number of virtual shards will be several times (user-defined) more than the number of servers. Note: for additional data spreading, you can impose some spreading transformation on the sharding key. Not thought out. For example, instead of a key (CounterID, Date, UserID) use for sharding (hash (UserID)% 10, CounterID, Date, UserID). But in this case, even small CounterIDs will fall into 10 ranges. +4. Similarly. +5. If several virtual shards are located on a single server, their replicas will be spread over a larger number of servers, and during recovery, there will be more fanout. +6. Small requests will use one shard. While large requests will use several shards on the same server. But since each shard will be somewhat smaller, the data in the MergeTree table will probably be presented by a smaller set of parts. For example, we now have a maximum part size of 150 GiB, and for large tables, many such large chunks are formed in one partition. And if there are several tables, there will be a smaller number of large chunks in each. On the other hand, when inserting data, a larger number of small pieces will be generated on each server. And these small parts will cause an increase in the number of seeks. But not much, as the fresh data will be in the page cache. That is why too many virtual shards per server might not work well. +7. Pretty hard. You can have groups of neighboring shards on different disks of the same server. But then reading of medium size ranges will not be parallelized (since the whole range will be on one disk). In RAID, the problem is solved by the fact that the size of the chunk is relatively small (typically 1 megabyte). It would be possible to come up with a separate distribution of data in different pieces on different disks. But it is too difficult to design and implement carefully. Probably it“s better not to do the whole thing, and as a minimum, make it so that when on the JBOD server, one server disk is selected for the location of one shard. +8. It is possible to identify the sharding scheme with a string, which may be common to different tables. The criterion for splitting shards is determined based on the total amount of data for all tables with the same sharding scheme. +9. It is solved completely by changing the mapping of virtual shards on the servers. This mapping can be controlled independently of everything else. +10. Servers can cache the sharding map (both parts of it) for a while and update it usually asynchronously. When rebalancing data due to the splitting of virtual shards, you should keep the old data for a longer time. Similarly, when transferring replicas between servers. Upon request, the initiator server also asks if the remote server has the necessary data: data for the required shard according to the sharding scheme that is cached by the initiator server. For the query, one live replica of each shard is selected, on which there is data. If suddenly there were none, then it is worthwhile to update the sharding map synchronously, as for some reason all the replicas were transferred somewhere. +11. It is trivial. +12. It is solved on the basis that more than one shard accounts for one server and the fact that the distribution of shards replicas among servers is more or less arbitrary and can take into account the amount of disk space. +## Issues + +To ingest data into a table, you can send an INSERT query to any server. The data will be divided into ranges and recorded on the desired servers. At the same time, it is synchronously ensured that we use a fresh sharding map - it is requested before the data is inserted and it is checked that it is not out of date, simultaneously with the commit in ZK. + +When a SELECT query is used, if the old sharding map was used, the latest data will not be visible. Therefore, the asynchronous update interval of the sharding map for SELECT should be made customizable, and an option should be added to synchronously use the latest sharding map. + +For fairly large tables, it turns out that an INSERT request breaks the data into many small pieces and writes to all servers (example: with 500 servers, you need to commit 5000 replicas of shards). This should work since the probability of inaccessibility or inhibition of all replicas of one shard is still low. But it will work slowly and, possibly, unstable. With a lot of INSERTs, there will be a terrible load on the coordinator. Although it can withstand one INSERT per second normally. To achieve high throughput of INSERTs, it is sufficient to simply make them parallel, but with the same low frequency of INSERTs in general. However, this is still a big problem. + +There are the following possible solutions: + +1. You can add something to the beginning of the sharding key. For example, Date % 10 or toMinute. Then INSERTs will touch fewer shards (in the typical case when recent data is inserted), but at the same time during some time intervals, some shards will be hotter than others. Normally, if it reduces the number of active shards, for example, from 5000 on INSERT to 500. It is also very inconvenient for users. +2. You can come up with some kind of incomprehensible sharding scheme, where the fresh data first falls into some fresh shard where it is not clear where from where it is then lazily overwritten. A fresh shard is essentially a distributed queue. At the same time, a fresh shard with SELECT is always requested. Not so good. And still, it contradicts the atomicity of these transfers of data, visible at SELECT. Alternatively, you could relax the requirements if you allow SELECT not to see some of the fresh data. +It looks like it“s generally not working well at a cluster size of over 500 servers. +Another problem is that to properly spread the ranges of the primary key, the number of virtual shards must be no less than the number of servers squared. And this is too much. +How to Get Around These Issues +For sharding, you can add some more intermediate mappings. There are the following options: +1. Splitting each shard into a set of shards in an arbitrary way. For example, 10 pieces. This is equivalent to adding a random number 0.N-1 to the beginning of the sharding key, which means nothing. Then with INSERT, you can only insert into one randomly selected shard, or a minimum sized shard, or some kind of round-robin; and as a result, INSERT becomes easier. But this increases the fanout of all point SELECTs. For convenience, such a partition can be done dynamically - only large enough shards can be divided in such a way (this will help avoid excessive splitting of old shards in the case when the sharding key starts with Date and the data is inserted in the Date order) or do such a partition starting from the situation when the number of shards is large enough (restriction on top of fanout INSERT requests). +An additional advantage: in the case of servers with JBOD, it is possible to prefer to place such second-level shards on the disks of one server, which half emulates RAID-0. +But there is a serious drawback: there is no possibility to do local IN / JOIN. For example, this possibility is assumed if the sharding key is hash (UserID), and we do JOIN by UserID. It would be possible to avoid this drawback by always placing all the “symmetric” shards on one server. +2. A mapping that spreads the data while keeping the number of virtual shards. The essence of this mapping is as follows: + - The spreading factor is set, for example, `N = 10.` As the very first mapping, 10 times more ranges are generated. For example, if we want to end up with 7 shards, then we divide the data into 70 ranges. + - Then these ranges are renumbered in a circle with numbers from 0.6 and the ranges with the same number will fall into one shard, as a result, there will be 7 shards again. + - The continuous analogue of this mapping: `x in [0, 1) -> fractional_part (x * N)`, multiplication by N on a circle. + +If you draw it on the picture in Cartesian coordinates, you get a “saw” with 10 teeth. + +After this, it becomes obvious that this mapping simultaneously spreads the data and preserves its locality. + +See also: [Arnold's cat map](https://en.wikipedia.org/wiki/Arnold%27s_cat_map). + +But what is described here does not exactly work. First, until a sufficient amount of data has been accumulated, it is impossible to create a uniform division into parts (there is no place to count quantiles). Secondly, according to such a simple scheme, it is impossible to divide the intervals. + +There is an option in which, instead of dividing a range in half, it uses splitting into 4 parts, which are then mapped into two shards. It is also not clear how this will work. diff --git a/website/blog/en/2019/clickhouse-at-percona-live-2019.md b/website/blog/en/2019/clickhouse-at-percona-live-2019.md new file mode 100644 index 00000000000..15a9973c0d0 --- /dev/null +++ b/website/blog/en/2019/clickhouse-at-percona-live-2019.md @@ -0,0 +1,38 @@ +--- +title: 'ClickHouse at Percona Live 2019' +image: 'https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/main.jpg' +date: '2019-06-04' +tags: ['Percona Live','USA','Texas','Austin', 'events', 'conference'] +--- + +This year American episode of [Percona Live](https://www.percona.com/live/19/) took place in nice waterfront location in Austin, TX, which welcomed open source database experts with pretty hot weather. ClickHouse community is undeniably growing and it became a common database product to give a talk about or at least compare or refer to, while just [two short years ago](../2017/clickhouse-at-percona-live-2017.md) it was more like “wth is ClickHouse?”. + +Alexey Rubin from VirtualHealth compared two column-oriented databases: ClickHouse and MariaDB Column Store. Bottom line was no surprise, ClickHouse is noticeably faster and MariaDB is more familiar for MySQL users, details were useful though. +![Alexey Rubin from VirtualHealth](https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/1.jpg) + +Alexey Milovidov from Yandex have demonstrated how exactly ClickHouse became even faster in recent releases. +![Alexey Milovidov from Yandex](https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/2.jpg) + +Alexander Zaitsev and Robert Hodges from Altinity have given an entry level tutorial to ClickHouse, which included loading in demo dataset and going through realistic queries against it with some extra variation demonstrating possible query optimization techniques. [Slides](https://www.percona.com/live/19/sites/default/files/slides/Making%20HTAP%20Real%20with%20TiFlash%20--%20A%20TiDB%20Native%20Columnar%20Extension%20-%20FileId%20-%20174070.pdf). Also Altinity was sponsoring the ClickHouse booth in Expo Hall which became an easy spot for people interested in ClickHouse to chat outside of talks. +![Alexander Zaitsev and Robert Hodges from Altinity](https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/3.jpg) + +Ruoxi Sun from PingCAP introduced TiFlash, column-oriented add-on to TiDB for analytics based on ClickHouse source code. Basically it provides [MergeTree](/docs/en/engines/table-engines/mergetree-family/mergetree/)-like table engine that is hooked up to TiDB replication and has in-memory row-friendly cache for recent updates. Unfortunately, PingCAP has no plans to bring TiFlash to opensource at the moment. [Slides](https://www.percona.com/live/19/sites/default/files/slides/Making%20HTAP%20Real%20with%20TiFlash%20--%20A%20TiDB%20Native%20Columnar%20Extension%20-%20FileId%20-%20174070.pdf). +![Ruoxi Sun from PingCAP](https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/4.jpg) + +ClickHouse has also been covered in talk by Jervin Real and Francisco Bordenave from Percona with overview of moving and replicating data around MySQL-compatible storage solutions. [Slides](https://www.percona.com/live/19/sites/default/files/slides/Replicating%20MySQL%20Data%20to%20TiDB%20For%20Real-Time%20Analytics%20-%20FileId%20-%20187672.pdf). +![Jervin Real](https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/5.jpg) + +ClickHouse represented columnar storage systems in venture beyond relational by Marcos Albe from Percona. +![Marcos Albe from Percona](https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/6.jpg) + +Jervin Real from Percona have demonstrated real case study of applying ClickHouse in practice. It heavily involved manual partitions manipulation, hopefully audience have understood that it is an option, but not exactly a best practice for most use cases. [Slides](https://www.percona.com/live/19/sites/default/files/slides/Low%20Cost%20Transactional%20and%20Analytics%20With%20MySQL%20and%20Clickhouse,%20Have%20Your%20Cake%20and%20Eat%20It%20Too!%20-%20FileId%20-%20187674.pdf). +![Jervin Real from Percona](https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/7.jpg) + +Evgeny Potapov from ITSumma went through modern options for time-series storage and once more confirmed ClickHouse is leading the way in this field as well. +![Evgeny Potapov from ITSumma](https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/8.jpg) + +Event location in the center of US provided equal opportunities for peoplefrom East and West Coast to show up, but presence of people from other countries was also quite noticeable. The content they all brought in was top notch as usual. +![The venue](https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/9.jpg) + +Austin after the Event. +![Austin](https://blog-images.clickhouse.tech/en/2019/clickhouse-at-percona-live-2019/10.jpg) diff --git a/website/blog/en/2019/clickhouse-lecture-at-institute-of-computing-technology-chinese-academy-of-science-on-june-11-2019.md b/website/blog/en/2019/clickhouse-lecture-at-institute-of-computing-technology-chinese-academy-of-science-on-june-11-2019.md new file mode 100644 index 00000000000..ab55f746bdd --- /dev/null +++ b/website/blog/en/2019/clickhouse-lecture-at-institute-of-computing-technology-chinese-academy-of-science-on-june-11-2019.md @@ -0,0 +1,17 @@ +--- +title: 'ClickHouse Lecture at Institute of Computing Technology, Chinese Academy of Science on June 11, 2019' +image: 'https://blog-images.clickhouse.tech/en/2019/clickhouse-lecture-at-institute-of-computing-technology-chinese-academy-of-science-on-june-11-2019/main.jpg' +tags: ['lecture', 'events', 'China', 'Beijing', 'university', 'academy', 'institute'] +date: '2019-06-14' +--- + +Alexey Milovidov, head of ClickHouse development group at Yandex, have given an open two-part lecture at [Institute of Computing Technology, Chinese Academy of Science](http://english.ict.cas.cn/): + +- ClickHouse history and evolution of Yandex.Metrica storage system +- Internal implementation of ClickHouse and reasoning behind design decisions + +The event has been organised by [Amos Bird](https://github.com/amosbird), who is one of the most active ClickHouse community members and contributors, at the same time being a last year PhD student at this institution. + +Alexey with the event announcement: +![Alexey with the event announcement](https://blog-images.clickhouse.tech/en/2019/clickhouse-lecture-at-institute-of-computing-technology-chinese-academy-of-science-on-june-11-2019/1.jpg) + diff --git a/website/blog/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019.md b/website/blog/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019.md new file mode 100644 index 00000000000..ace5967736c --- /dev/null +++ b/website/blog/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019.md @@ -0,0 +1,35 @@ +--- +title: 'ClickHouse Meetup in Beijing on June 8, 2019' +image: 'https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/main.jpg' +date: '2019-06-13' +tags: ['meetup','Beijing','China','events'] +--- + +24th ClickHouse Meetup globally and 3rd one in China took place in Beijing on Dragon Boat Festival weekend, which appeared to have a rich history and be a popular opportunity for Chinese people to travel around the country. Nevertheless the ClickHouse Meetup venue was more than full as usual, this time kindly provided by Gaea Mobile, with hundreds more people watching live broadcast online. Yandex ClickHouse team have extensively used this trip as an opportunity to strengthen the bond with ClickHouse Community in China by also giving an [open lecture in Institute of Computing Technology, Chinese Academy of Science](clickhouse-lecture-at-institute-of-computing-technology-chinese-academy-of-science-on-june-11-2019.md) and by having a private conversations with the most active local corporate users including ByteDance and JD.com. + +Welcome word by William Kwok, CTO of Analysys, who played the key role in making this event in particular possible and also in establishment of ClickHouse Community in China: +![William Kwok, CTO of Analysys](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/1.jpg) + +He's also administering ClickHouse WeChat groups, feel free to ask him for invite (@guodaxia2999 at WeChat): +![@guodaxia2999 at WeChat](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/2.jpg) + +Alexey Milovidov from ClickHouse core developers team at Yandex got the content part of main event part started with overview of new features and roadmap overview: +![Alexey Milovidov](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/3.jpg) + +Amos Bird, one of the most active ClickHouse contributors either in China and worldwide, shares his experience of using ClickHouse for graph processing ([slides](https://github.com/ClickHouse/clickhouse-presentations/raw/master/meetup24/2.%20SQLGraph%20--%20When%20ClickHouse%20marries%20graph%20processing%20Amoisbird.pdf)): +![Amos Bird](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/4.jpg) + +Yan Gangqiang from Golden Data shares details of their approach to data storage for surveys system based on ClickHouse ([slides](https://presentations.clickhouse.tech/meetup24/3.%20金数据数据架构调整方案Public.pdf)): +![Yan Gangqiang](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/5.jpg) + +ClickHouse for beginners talk by Percent ([slides](https://presentations.clickhouse.tech/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf)): +![Percent](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/6.jpg) + +ClickHouse core developer Nikolay Kochetov demonstrates upcoming query execution pipeline changes ([slides](https://presentations.clickhouse.tech/meetup24/5.%20Clickhouse%20query%20execution%20pipeline%20changes/)): +![Nikolay Kochetov](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/7.jpg) + +Pre-meetup meeting with active ClickHouse community members in China: +![Pre-meetup meeting](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/8.jpg) + +ClickHouse branded Beijing duck :) +![ClickHouse branded Beijing duck](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-beijing-on-june-8-2019/9.jpg) diff --git a/website/blog/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019.md b/website/blog/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019.md new file mode 100644 index 00000000000..71176ab3d47 --- /dev/null +++ b/website/blog/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019.md @@ -0,0 +1,41 @@ +--- +title: 'ClickHouse Meetup in Limassol on May 7, 2019' +image: 'https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019/main.jpg' +date: '2019-05-14' +tags: ['meetup', 'Cyprus', 'Limassol', 'events'] +--- + +The first open-air ClickHouse Meetup took place in the heart of Limassol, the second-largest city of Cyprus, on the roof kindly provided by Exness Group. The views were stunning, but speakers did a great job competing with them for audience attention. Over one hundred people have joined in, which once again confirms high interest in ClickHouse around the globe. Meetup content is also available as [video recording](https://www.youtube.com/watch?v=_rpU-TvSfZ8). + +![Intro](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019/1.jpg) + +[Kirill Shvakov](https://github.com/kshvakov) has played the key role in making this event possible by reaching out to the ClickHouse Community at Cyprus, finding the great venue, and other speakers. Most of the worldwide ClickHouse Meetups happen thanks to active community members like Kirill, if you want to help us organize ClickHouse Meetup in your area, please reach the Yandex ClickHouse team via [this form](https://clickhouse.tech/#meet) or any other convenient way. + +![Kirill Shvakov](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019/2.jpg) + +Kirill is well known for his top-notch [ClickHouse Go Driver](https://github.com/clickhouse/clickhouse-go) running over native protocol, but his opening talk was about his experience optimizing ClickHouse queries and solving real-world tasks at Integros and Wisebits. [Slides](https://presentations.clickhouse.tech/meetup22/strategies.pdf). [Full query listings](https://github.com/kshvakov/ClickHouse-Meetup-Exness). + +The event has begun in the early evening... +![Evening in Limassol](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019/3.jpg) + +...but it took just around one hour for nature to turn the night mode on. It made the projected slides easier to read. +![Night in Limassol](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019/4.jpg) + +Sergey Tomilov with his colleagues from the Exness Platform Team has shared details on the evolution of their systems for analyzing logs and metrics and how they ended up relying on ClickHouse for long-term storage ([slides](https://presentations.clickhouse.tech/meetup22/exness.pdf)): +![Sergey Tomilov](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019/5.jpg) + +Alexey Milovidov from the Yandex ClickHouse team has demonstrated features from recent ClickHouse releases and gave an update on what's coming soon ([slides](https://presentations.clickhouse.tech/meetup22/new_features/)): +![Alexey Milovidov](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019/6.jpg) + +Alexander Zaitsev, CTO of Altinity, have shown an overview of how to integrate ClickHouse into environments running on Kubernetes ([slides](https://presentations.clickhouse.tech/meetup22/kubernetes.pdf)): +![Alexander Zaitsev, CTO of Altinity](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019/7.jpg) + +Vladimir Goncharov, a backend engineer from Aloha Browser, has closed the ClickHouse Limassol Meetup by demonstrating few projects that allow integrating other opensource tools for logs processing with ClickHouse ([slides](https://presentations.clickhouse.tech/meetup22/aloha.pdf)): +![Vladimir Goncharov](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019/8.jpg) + +Unfortunately, midnight was closing in and only the most weather-proof ClickHouse fans have managed to stay the whole event as it started getting pretty chilly. + +![Final](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-limassol-on-may-7-2019/9.jpg) + +More photos from the event are available at [short event afterword by Exness](https://www.facebook.com/events/386638262181785/permalink/402167077295570/). + diff --git a/website/blog/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019.md b/website/blog/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019.md new file mode 100644 index 00000000000..6141b4c2fd5 --- /dev/null +++ b/website/blog/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019.md @@ -0,0 +1,28 @@ +--- +title: 'ClickHouse Meetup in Madrid on April 2, 2019' +image: 'https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019/main.jpg' +date: '2019-04-11' +tags: ['meetup', 'Spain', 'Madrid', 'events'] +--- + +Madrid ClickHouse Meetup has probably been the largest one in the EU so far with well over one hundred attendees. As usual, we've seen not only people working and living in the same city, Madrid, but also many people who have traveled a long way to join the event and talk about ClickHouse use cases and learn about new and upcoming features. + +Opening word by [Javi Santana](https://twitter.com/javisantana), who personally made this event possible by gathering up all the people and setting up the venue provided by Google Campus for Startups: +![Javi Santana](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019/1.jpg) + +Alexander Zaitsev, CTO of Altinity, has introduced ClickHouse to those who're just starting to use it or only considering for future ([slides](https://www.slideshare.net/Altinity/clickhouse-introduction-by-alexander-zaitsev-altinity-cto)): +![Alexander Zaitsev](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019/2.jpg) + +Robert Hodges, CEO of Altinity, has probably traveled the longest distance to join the event since he's based in California and he has also [published his thoughts on this event in the Altinity blog](https://www.altinity.com/blog/2019/4/9/madrid-clickhouse-meetup-summary). + +Alexey Milovidov from Yandex has shown the recent advancements in ClickHouse features and briefly walked the audience through the current roadmap ([slides](https://presentations.clickhouse.tech/meetup21/new_features/)): +![Alexey Milovidov from Yandex](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019/3.jpg) + +Iago Enriquez from Idealista was talking about their migration from “legacy” commercial DBMS to ClickHouse. It was the first time we've heard that someone talking about using two flagship opensource products by Yandex together in production. They are using [CatBoost](https://catboost.ai/) model inference right from ClickHouse SQL queries to fill in the incompleteness of their source data. Unfortunately, slides of Iago's talk were not allowed to be published. +![Iago Enriquez from Idealista](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019/4.jpg) + +David Pardo Villaverde from Corunet gave a talk about how they've used ClickHouse to prepare data for dense model generation for one of their clients. It took a pretty short time on a single server. Fun quote from conclusions: “If I wasn't already married, I'd marry it! \[ClickHouse\]” ([slides](https://presentations.clickhouse.tech/meetup21/predictive_models.pdf)): +![David Pardo Villaverde from Corunet](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019/5.jpg) + +Closing talk of the meetup was by Murat Kabilov fro Adjust Gmbh, he was demonstrating his opensource project [pg2ch](https://github.com/mkabilov/pg2ch) that allows to sync data from PostgreSQL to ClickHouse in real-time ([slides](https://presentations.clickhouse.tech/meetup21/postgres_to_clickhouse.pdf)). +![Murat Kabilov fro Adjust Gmbh](https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-madrid-on-april-2-2019/6.jpg) diff --git a/website/blog/en/2019/clickhouse-meetup-in-san-francisco-on-june-4-2019.md b/website/blog/en/2019/clickhouse-meetup-in-san-francisco-on-june-4-2019.md new file mode 100644 index 00000000000..94ad125b71b --- /dev/null +++ b/website/blog/en/2019/clickhouse-meetup-in-san-francisco-on-june-4-2019.md @@ -0,0 +1,10 @@ +--- +title: 'ClickHouse Meetup in San Francisco on June 4, 2019' +image: 'https://blog-images.clickhouse.tech/en/2019/clickhouse-meetup-in-san-francisco-on-june-4-2019/main.jpg' +date: '2019-06-12' +tags: ['meetup','USA','San Francisco','events', 'California', 'Bay Area'] +--- + +23th ClickHouse Meetup in San Francisco was held in CloudFlare office and co-organized by Altinity. There were about 35 attendees, most of them are experienced ClickHouse users from SF and Bay Area. The meetup started with an introduction by Robert Hodges, Altinity CEO and continued with a lightning talk by Alan Braithwaite from Segment.com about their experience with ClickHouse. Next talk from Alexander Zaitsev about ClickHouse operator for Kubernetes gained much attention from the audience because Kubernetes is in fact very popular even for databases. At the end there was a presentation from the ClickHouse developer Alexey Milovidov about new and upcoming features with a roadmap. There was a discussion about the details of implementation and design of the most appreciated features. We were happy to meet with ClickHouse contributors at the meetup. Slides from the event are [available on GitHub](https://github.com/clickhouse/clickhouse-presentations/tree/master/meetup23). + +As we see increasing demand for ClickHouse events in SF and Bay Area, we have already started planning the next event. diff --git a/website/blog/en/2019/how-to-speed-up-lz4-decompression-in-clickhouse.md b/website/blog/en/2019/how-to-speed-up-lz4-decompression-in-clickhouse.md new file mode 100644 index 00000000000..b5799702d04 --- /dev/null +++ b/website/blog/en/2019/how-to-speed-up-lz4-decompression-in-clickhouse.md @@ -0,0 +1,12 @@ +--- +title: 'How to speed up LZ4 decompression in ClickHouse?' +image: 'https://blog-images.clickhouse.tech/en/2019/how-to-speed-up-lz4-decompression-in-clickhouse/main.jpg' +date: '2019-06-25' +tags: ['performance', 'lz4', 'article', 'decompression'] +--- + +When you run queries in [ClickHouse](https://clickhouse.tech/), you might notice that the profiler often shows the `LZ_decompress_fast` function near the top. What is going on? This question had us wondering how to choose the best compression algorithm. + +ClickHouse stores data in compressed form. When running queries, ClickHouse tries to do as little as possible, in order to conserve CPU resources. In many cases, all the potentially time-consuming computations are already well optimized, plus the user wrote a well thought-out query. Then all that's left to do is to perform decompression. + +[Read further](https://habr.com/en/company/yandex/blog/457612/) diff --git a/website/blog/en/2019/schedule-of-clickhouse-meetups-in-china-for-2019.md b/website/blog/en/2019/schedule-of-clickhouse-meetups-in-china-for-2019.md new file mode 100644 index 00000000000..726d714b765 --- /dev/null +++ b/website/blog/en/2019/schedule-of-clickhouse-meetups-in-china-for-2019.md @@ -0,0 +1,14 @@ +--- +title: 'Schedule of ClickHouse Meetups in China for 2019' +image: 'https://blog-images.clickhouse.tech/en/2019/schedule-of-clickhouse-meetups-in-china-for-2019/main.jpg' +date: '2019-04-18' +tags: ['China', 'Beijing', 'Shanghai', 'Shenzhen', 'announcement', 'meetup'] +--- + +Last year there were two ClickHouse Meetups in Beijing, in [January](../2018/clickhouse-community-meetup-in-beijing-on-january-27-2018.md) and [October](../2018/clickhouse-community-meetup-in-beijing-on-january-27-2018.md), and they appeared to be in extremely high demand, with fully packed venue and thousands of people watching online. So this year we decided to try to expand meetups to other large cities in China where we see the most interest in ClickHouse based on website visits. Here's the current schedule and sign up pages: + +- [ClickHouse Community Meetup in Beijing](https://www.huodongxing.com/event/2483759276200) on June 8. +- [ClickHouse Community Meetup in Shenzhen](https://www.huodongxing.com/event/3483759917300) on October 20. +- [ClickHouse Community Meetup in Shanghai](https://www.huodongxing.com/event/4483760336000) on October 27. + +到时候那里见！ diff --git a/website/blog/en/2020/five-methods-for-database-obfuscation.md b/website/blog/en/2020/five-methods-for-database-obfuscation.md new file mode 100644 index 00000000000..9a6615b0079 --- /dev/null +++ b/website/blog/en/2020/five-methods-for-database-obfuscation.md @@ -0,0 +1,10 @@ +--- +title: 'Five Methods For Database Obfuscation' +image: 'https://blog-images.clickhouse.tech/en/2020/five-methods-for-database-obfuscation/main.jpg' +date: '2020-01-27' +tags: ['article', 'obfuscation'] +--- + +ClickHouse users already know that its biggest advantage is its high-speed processing of analytical queries. But claims like this need to be confirmed with reliable performance testing. + +[Read further](https://habr.com/en/company/yandex/blog/485096/) diff --git a/website/blog/en/index.md b/website/blog/en/index.md new file mode 100644 index 00000000000..227a69408dc --- /dev/null +++ b/website/blog/en/index.md @@ -0,0 +1,3 @@ +--- +is_index: true +--- diff --git a/website/blog/en/redirects.txt b/website/blog/en/redirects.txt new file mode 100644 index 00000000000..80a57d38ebc --- /dev/null +++ b/website/blog/en/redirects.txt @@ -0,0 +1,32 @@ +clickhouse-meetup-in-berlin-october-5-2017.md 2017/clickhouse-meetup-in-berlin-october-5-2017.md +clickhouse-meetup-at-berlin-october-5-2017.md 2017/clickhouse-meetup-in-berlin-october-5-2017.md +clickhouse-meetup-in-santa-clara-may-4-2017.md 2017/clickhouse-meetup-in-santa-clara-may-4-2017.md +clickhouse-meetup-at-santa-clara-may-4-2017.md 2017/clickhouse-meetup-in-santa-clara-may-4-2017.md +clickhouse-community-meetup-in-berlin-on-july-3.md 2018/announcing-clickhouse-community-meetup-in-berlin-on-july-3.md +evolution-of-data-structures-in-yandex-metrica.md 2016/evolution-of-data-structures-in-yandex-metrica.md +how-to-update-data-in-clickhouse.md 2016/how-to-update-data-in-clickhouse.md +yandex-opensources-clickhouse.md 2016/yandex-opensources-clickhouse.md +clickhouse-at-data-scale-2017.md 2017/clickhouse-at-data-scale-2017.md +clickhouse-meetup-in-berlin-october-5-2017.md 2017/clickhouse-meetup-in-berlin-october-5-2017.md +join-the-clickhouse-meetup-in-berlin.md 2017/join-the-clickhouse-meetup-in-berlin.md +clickhouse-at-percona-live-2017.md 2017/clickhouse-at-percona-live-2017.md +clickhouse-meetup-in-santa-clara-may-4-2017.md 2017/clickhouse-meetup-in-santa-clara-may-4-2017.md +announcing-clickhouse-meetup-in-amsterdam-on-november-15.md 2018/announcing-clickhouse-meetup-in-amsterdam-on-november-15.md +clickhouse-community-meetup-in-berlin-on-july-3-2018.md 2018/clickhouse-community-meetup-in-berlin-on-july-3-2018.md +clickhouse-at-analysys-a10-2018.md 2018/clickhouse-at-analysys-a10-2018.md +clickhouse-community-meetup-in-berlin-on-july-3.md 2018/clickhouse-community-meetup-in-berlin-on-july-3.md +clickhouse-community-meetup-in-paris-on-october-2-2018.md 2018/clickhouse-community-meetup-in-paris-on-october-2-2018.md +clickhouse-community-meetup-in-beijing-on-october-28-2018.md 2018/clickhouse-community-meetup-in-beijing-on-october-28-2018.md +clickhouse-meetup-in-amsterdam-on-november-15-2018.md 2018/clickhouse-meetup-in-amsterdam-on-november-15-2018.md +clickhouse-community-meetup-in-beijing-on-january-27-2018.md 2018/clickhouse-community-meetup-in-beijing-on-january-27-2018.md +clickhouse-at-percona-live-europe-2018.md 2018/clickhouse-at-percona-live-europe-2018.md +concept-cloud-mergetree-tables.md 2018/concept-cloud-mergetree-tables.md +clickhouse-meetup-in-limassol-on-may-7-2019.md 2019/clickhouse-meetup-in-limassol-on-may-7-2019.md +schedule-of-clickhouse-meetups-in-china-for-2019.md 2019/schedule-of-clickhouse-meetups-in-china-for-2019.md +clickhouse-lecture-at-institute-of-computing-technology-chinese-academy-of-science-on-june-11-2019.md 2019/clickhouse-lecture-at-institute-of-computing-technology-chinese-academy-of-science-on-june-11-2019.md +clickhouse-meetup-in-san-francisco-on-june-4-2019.md 2019/clickhouse-meetup-in-san-francisco-on-june-4-2019.md +how-to-speed-up-lz4-decompression-in-clickhouse.md 2019/how-to-speed-up-lz4-decompression-in-clickhouse.md +clickhouse-at-percona-live-2019.md 2019/clickhouse-at-percona-live-2019.md +clickhouse-meetup-in-madrid-on-april-2-2019.md 2019/clickhouse-meetup-in-madrid-on-april-2-2019.md +clickhouse-meetup-in-beijing-on-june-8-2019.md 2019/clickhouse-meetup-in-beijing-on-june-8-2019.md +five-methods-for-database-obfuscation.md 2020/five-methods-for-database-obfuscation.md diff --git a/website/blog/ru/2016/clickhouse-meetup-v-moskve-21-noyabrya-2016.md b/website/blog/ru/2016/clickhouse-meetup-v-moskve-21-noyabrya-2016.md new file mode 100644 index 00000000000..71fb8da8215 --- /dev/null +++ b/website/blog/ru/2016/clickhouse-meetup-v-moskve-21-noyabrya-2016.md @@ -0,0 +1,8 @@ +--- +title: 'ClickHouse Meetup в Москве, 21 ноября 2016' +image: 'https://blog-images.clickhouse.tech/ru/2016/clickhouse-meetup-v-moskve-21-noyabrya-2016/main.jpg' +date: '2016-11-22' +tags: ['мероприятия', 'meetup', 'Москва'] +--- + +[Посмотреть видео](https://events.yandex.ru/lib/talks/4351/) diff --git a/website/blog/ru/2016/clickhouse-na-highload-2016.md b/website/blog/ru/2016/clickhouse-na-highload-2016.md new file mode 100644 index 00000000000..fb950db35b0 --- /dev/null +++ b/website/blog/ru/2016/clickhouse-na-highload-2016.md @@ -0,0 +1,14 @@ +--- +title: 'ClickHouse на HighLoad++ 2016' +image: 'https://blog-images.clickhouse.tech/ru/2016/clickhouse-na-highload-2016/main.jpg' +date: '2016-12-10' +tags: ['мероприятия', 'конференции', 'Москва', 'HighLoad++'] +--- + +![iframe](https://www.youtube.com/embed/TAiCXHgZn50) + +[Расшифровка доклада](https://habrahabr.ru/post/322724/) + +![iframe](https://www.youtube.com/embed/tf38TPvwjJ4) + +[Расшифровка доклада](https://habrahabr.ru/post/322620/) diff --git a/website/blog/ru/2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md b/website/blog/ru/2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md new file mode 100644 index 00000000000..6404ee0465a --- /dev/null +++ b/website/blog/ru/2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md @@ -0,0 +1,10 @@ +--- +title: 'ClickHouse на встрече про инфраструктуру хранения и обработки данных в Яндексе' +image: 'https://blog-images.clickhouse.tech/ru/2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse/main.jpg' +date: '2016-10-16' +tags: ['мероприятия', 'инфраструктура'] +--- + +![iframe](https://www.youtube.com/embed/Ho4_dQk7dAg) + +[Страница мероприятия «Яндекс изнутри: инфраструктура хранения и обработки данных»](https://events.yandex.ru/events/meetings/15-oct-2016/), прошедшего 15 октября 2016 года. diff --git a/website/blog/ru/2016/yandeks-otkryvaet-clickhouse.md b/website/blog/ru/2016/yandeks-otkryvaet-clickhouse.md new file mode 100644 index 00000000000..36daa047ce7 --- /dev/null +++ b/website/blog/ru/2016/yandeks-otkryvaet-clickhouse.md @@ -0,0 +1,10 @@ +--- +title: 'Яндекс открывает ClickHouse' +image: 'https://blog-images.clickhouse.tech/ru/2016/yandeks-otkryvaet-clickhouse/main.jpg' +date: '2016-06-15' +tags: ['анонс', 'GitHub', 'лицензия'] +--- + +Сегодня внутренняя разработка компании Яндекс — [аналитическая СУБД ClickHouse](https://clickhouse.tech/), стала доступна каждому. Исходники опубликованы на [GitHub](https://github.com/yandex/ClickHouse) под лицензией Apache 2.0. + +ClickHouse позволяет выполнять аналитические запросы в интерактивном режиме по данным, обновляемым в реальном времени. Система способна масштабироваться до десятков триллионов записей и петабайт хранимых данных. Использование ClickHouse открывает возможности, которые раньше было даже трудно представить: вы можете сохранять весь поток данных без предварительной агрегации и быстро получать отчёты в любых разрезах. ClickHouse разработан в Яндексе для задач [Яндекс.Метрики](https://metrika.yandex.ru/) — второй по величине системы веб-аналитики в мире. diff --git a/website/blog/ru/2017/clickhouse-meetup-edet-v-minsk.md b/website/blog/ru/2017/clickhouse-meetup-edet-v-minsk.md new file mode 100644 index 00000000000..fb84a16c02a --- /dev/null +++ b/website/blog/ru/2017/clickhouse-meetup-edet-v-minsk.md @@ -0,0 +1,14 @@ +--- +title: 'ClickHouse MeetUp едет в Минск!' +image: 'https://blog-images.clickhouse.tech/ru/2017/clickhouse-meetup-edet-v-minsk/main.jpg' +date: '2017-06-13' +tags: ['мероприятия', 'meetup', 'Минск', 'Беларусь', 'анонс'] +--- + +29 июня в Минске впервые выступят с докладами создатели СУБД ClickHоuse и те, кто ежедневно использует её для решения аналитических задач. Докладчики расскажут о последних изменениях и предстоящих обновлениях СУБД, а также о нюансах работы с ней. + +Встреча будет интересна администраторам ClickHouse и тем, кто пока только присматривается к системе. Мы приглашаем белорусских пользователей также поделиться своим опытом использования ClickHоuse и выступить на встрече с блиц-докладами: при регистрации мы предложим вам такую возможность! + +Участие в мероприятии бесплатное, но необходимо заранее зарегистрироваться: количество мест в зале ограничено. + +Посмотреть программу и подать заявку на участие можно на [странице встречи](https://events.yandex.ru/events/meetings/29-june-2017). diff --git a/website/blog/ru/2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017.md b/website/blog/ru/2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017.md new file mode 100644 index 00000000000..80d399203b0 --- /dev/null +++ b/website/blog/ru/2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017.md @@ -0,0 +1,8 @@ +--- +title: 'ClickHouse Meetup в Екатеринбурге, 16 мая 2017' +image: 'https://blog-images.clickhouse.tech/ru/2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017/main.jpg' +date: '2017-05-17' +tags: ['мероприятия', 'meetup', 'Екатеринбург'] +--- + +[Посмотреть презентацию](https://presentations.clickhouse.tech/meetup6/) diff --git a/website/blog/ru/2017/clickhouse-meetup-v-minske-itogi.md b/website/blog/ru/2017/clickhouse-meetup-v-minske-itogi.md new file mode 100644 index 00000000000..de38df47af3 --- /dev/null +++ b/website/blog/ru/2017/clickhouse-meetup-v-minske-itogi.md @@ -0,0 +1,16 @@ +--- +title: 'ClickHouse MeetUp в Минске: итоги' +image: 'https://blog-images.clickhouse.tech/ru/2017/clickhouse-meetup-v-minske-itogi/main.jpg' +date: '2017-06-19' +tags: ['мероприятия', 'meetup', 'Минск', 'Беларусь'] +--- + +Недавно в Минске мы встретились с пользователями ClickHouse и техническими специалистами, кто только знакомится с СУБД. + +Мы делимся с вами презентациями докладчиков и будем рады ответить на вопросы в [чате ClickHouse в Телеграме](https://t.me/clickhouse_ru). + +[История создания ClickHouse, новости и планы по развитию](https://presentations.clickhouse.tech/meetup7/), Алексей Миловидов + +[Использование ClickHouse для мониторинга связности сети](https://presentations.clickhouse.tech/meetup7/netmon.pdf), Дмитрий Липин + +[Разбираемся во внутреннем устройстве ClickHouse](https://presentations.clickhouse.tech/meetup7/internals.pdf), Виталий Людвиченко diff --git a/website/blog/ru/2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017.md b/website/blog/ru/2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017.md new file mode 100644 index 00000000000..e4a614befad --- /dev/null +++ b/website/blog/ru/2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017.md @@ -0,0 +1,10 @@ +--- +title: 'ClickHouse Meetup в Новосибирске, 3 апреля 2017' +image: 'https://blog-images.clickhouse.tech/ru/2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017/main.jpg' +date: '2017-04-04' +tags: ['мероприятия', 'meetup', 'Новосибирск'] +--- + +[Презентация Алексея Миловидова](https://presentations.clickhouse.tech/meetup4/) + +[Презентация Марии Мансуровой](https://presentations.clickhouse.tech/meetup4/clickhouse_for_analysts.pdf) diff --git a/website/blog/ru/2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md b/website/blog/ru/2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md new file mode 100644 index 00000000000..3bdfd2763b8 --- /dev/null +++ b/website/blog/ru/2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md @@ -0,0 +1,8 @@ +--- +title: 'ClickHouse Meetup в Санкт-Петербурге, 28 февраля 2017' +image: 'https://blog-images.clickhouse.tech/ru/2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017/main.jpg' +date: '2017-03-01' +tags: ['мероприятия', 'meetup', 'Санкт-Петербург'] +--- + +![iframe](https://www.youtube.com/embed/CVrwp4Zoex4) diff --git a/website/blog/ru/2017/clickhouse-na-uwdc-2017.md b/website/blog/ru/2017/clickhouse-na-uwdc-2017.md new file mode 100644 index 00000000000..7b801181803 --- /dev/null +++ b/website/blog/ru/2017/clickhouse-na-uwdc-2017.md @@ -0,0 +1,10 @@ +--- +title: 'ClickHouse на UWDC 2017' +image: 'https://blog-images.clickhouse.tech/ru/2017/clickhouse-na-uwdc-2017/main.jpg' +date: '2017-05-20' +tags: ['мероприятия', 'конференции', 'Челябинск'] +--- + +![iframe](https://www.youtube.com/embed/isYA4e5zg1M?t=2h8m15s) + +[Посмотреть презентацию](https://presentations.clickhouse.tech/uwdc/) diff --git a/website/blog/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019.md b/website/blog/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019.md new file mode 100644 index 00000000000..38e697d6b4c --- /dev/null +++ b/website/blog/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019.md @@ -0,0 +1,38 @@ +--- +title: 'ClickHouse Meetup в Лимассоле, 7 мая 2019' +image: 'https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/main.jpg' +date: '2019-05-14' +tags: ['мероприятия', 'meetup', 'Лимассол', 'Кипр', 'Европа'] +--- + +Первый ClickHouse Meetup под открытым небом прошел в сердце Лимассола, второго по размеру города Кипра, на крыше, любезно предоставленной Exness Group. С крыши открывались сногсшибательные виды, но докладчики отлично справлялись с конкуренцией с ними за внимание аудитории. Более ста человек присоединилось к мероприятие, что в очередной раз подтверждает высокий интерес к ClickHouse по всему земному шару. Контент мероприятия также доступен в формате [видеозаписи](https://www.youtube.com/watch?v=_rpU-TvSfZ8). + +[Кирилл Шваков](https://github.com/kshvakov) сыграл ключевую роль в том, чтобы данное мероприятие стало возможным: наладил коммуникацию с ClickHouse сообществом на Кипре, нашел отличную площадку и докладчиков. Большинство ClickHouse митапов по всему миру происходят благодаря активным участникам сообщества таким как Кирилл. Если вы хотите помочь нам организовать ClickHouse митап в своём регионе, пожалуйста свяжитесь с командой ClickHouse в Яндексе через [эту форму](https://clickhouse.tech/#meet) или любым другим удобным способом. + +![Кирилл Шваков](https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/1.jpg) + +Кирилл широко известен благодаря его замечательногму [ClickHouse Go Driver](https://github.com/clickhouse/clickhouse-go), работающему по нативному протоколу, а его открывающий доклад был о его опыте оптимизации ClickHouse запросов и решению реальных прикладных задач в Integros и Wisebits. [Слайды](https://presentations.clickhouse.tech/meetup22/strategies.pdf). [Полные тексты запросов](https://github.com/kshvakov/ClickHouse-Meetup-Exness). + +Мероприятие началось ранним вечером… +![Вечер в Лимассоле](https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/2.jpg) + +…но природе потребовалось всего около часа, чтобы включить «ночной режим». Зато проецируемые слайды стало заметно легче читать. +![Ночь в Лимассоле](https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/3.jpg) + +Сергей Томилов с его коллегами из Exness Platform Team поделились деталями об эволюции их систем для анализа логов и метрик, а также как они в итоге стали использовать ClickHouse для долгосрочного хранения и анализа данных([слайды](https://presentations.clickhouse.tech/meetup22/exness.pdf)): +![Сергей Томилов](https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/4.jpg) + +Алексей Миловидов из команды ClickHouse в Яндексе продемонстрировал функциональность из недавних релизов ClickHouse, а также рассказал о том, что стоит ждать в ближайшем будущем([слайды](https://presentations.clickhouse.tech/meetup22/new_features/)): +![Алексей Миловидов](https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/5.jpg) + +Александр Зайцев, технический директор Altinity, показал обзор того, как можно интегрировать ClickHouse в окружения, работающие на Kubernetes([слайды](https://presentations.clickhouse.tech/meetup22/kubernetes.pdf)): +![Александр Зайцев](https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/6.jpg) + +Владимир Гончаров, бекенд разработчик из Aloha Browser, закрывал ClickHouse Limassol Meetup демонстрацией нескольких проектов для интеграции других opensource продуктов для анализа логов с ClickHouse ([слайды](https://presentations.clickhouse.tech/meetup22/aloha.pdf)): +![Владимир Гончаров](https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/7.jpg) + +К сожалению, приближалась полнось и только самые «морозостойкие» любители ClickHouse продержались всё мероприятие, так стало заметно холодать. + +![Лимассол](https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/8.jpg) + +Больше фотографий с мероприятия доступно в [коротком послесловии от Exness](https://www.facebook.com/events/386638262181785/permalink/402167077295570/). diff --git a/website/blog/ru/2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019.md b/website/blog/ru/2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019.md new file mode 100644 index 00000000000..d3a5471b1a8 --- /dev/null +++ b/website/blog/ru/2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019.md @@ -0,0 +1,10 @@ +--- +title: 'ClickHouse Meetup в Москве, 5 сентября 2019' +image: 'https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019/main.jpg' +date: '2019-09-06' +tags: ['мероприятия', 'meetup', 'Москва'] +--- + +![iframe](https://www.youtube.com/embed/videoseries?list=PL0Z2YDlm0b3gYSwohnKFUozYy9QdUpcT_) + +[Слайды опубликованы на GitHub](https://github.com/clickhouse/clickhouse-presentations/tree/master/meetup28). diff --git a/website/blog/ru/2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019.md b/website/blog/ru/2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019.md new file mode 100644 index 00000000000..d1dafe580f1 --- /dev/null +++ b/website/blog/ru/2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019.md @@ -0,0 +1,12 @@ +--- +title: 'ClickHouse Meetup в Новосибирске, 26 июня 2019' +image: 'https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019/main.jpg' +date: '2019-06-05' +tags: ['мероприятия', 'meetup', 'Новосибирск'] +--- + +Изюминкой второго ClickHouse митапа в Новосибирске были два низкоуровневых доклада с погружением во внутренности ClickHouse, а остальная часть контента была очень прикладной с конкретными сценариями. Любезно предоставленный S7 зал на сто человек был полон до самого завершения последнего доклада где-то ближе к полуночи. + +![iframe](https://www.youtube.com/embed/videoseries?list=PL0Z2YDlm0b3ionSVt-NYC9Vu_83xxhb4J) + +Как обычно, [все слайды опубликованы на GitHub](https://presentations.clickhouse.tech/meetup25). diff --git a/website/blog/ru/2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md b/website/blog/ru/2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md new file mode 100644 index 00000000000..8f8f9b4aae2 --- /dev/null +++ b/website/blog/ru/2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md @@ -0,0 +1,10 @@ +--- +title: 'ClickHouse Meetup в Санкт-Петербурге, 27 июля 2019' +image: 'https://blog-images.clickhouse.tech/ru/2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019/main.jpg' +date: '2019-08-01' +tags: ['мероприятия', 'meetup', 'Санкт-Петербург'] +--- + +![iframe](https://www.youtube.com/embed/videoseries?list=PL0Z2YDlm0b3j3X7TWrKmnEPcfEG901W-T) + +[Слайды опубликованы на GitHub](https://github.com/yandex/clickhouse-presentations/tree/master/meetup27). diff --git a/website/blog/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019.md b/website/blog/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019.md new file mode 100644 index 00000000000..cbd9e6c01fb --- /dev/null +++ b/website/blog/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019.md @@ -0,0 +1,12 @@ +--- +title: 'ClickHouse Meetup в Минске, 11 июля 2019' +image: 'https://blog-images.clickhouse.tech/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019/main.jpg' +date: '2019-07-12' +tags: ['мероприятия', 'meetup', 'Минск', 'Беларусь'] +--- + +![iframe](https://www.youtube.com/embed/videoseries?list=PL0Z2YDlm0b3hLz6dmyu6gM_X871FG9eCc) + +[Все слайды опубликованы на GitHub](https://github.com/yandex/clickhouse-presentations/tree/master/meetup26). + +![Минск](https://blog-images.clickhouse.tech/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019/1.jpg) diff --git a/website/blog/ru/index.md b/website/blog/ru/index.md new file mode 100644 index 00000000000..227a69408dc --- /dev/null +++ b/website/blog/ru/index.md @@ -0,0 +1,3 @@ +--- +is_index: true +--- diff --git a/website/blog/ru/redirects.txt b/website/blog/ru/redirects.txt new file mode 100644 index 00000000000..4e34d53af3d --- /dev/null +++ b/website/blog/ru/redirects.txt @@ -0,0 +1,15 @@ +yandeks-otkryvaet-clickhouse.md 2016/yandeks-otkryvaet-clickhouse.md +clickhouse-meetup-v-moskve-21-noyabrya-2016.md 2016/clickhouse-meetup-v-moskve-21-noyabrya-2016.md +clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md 2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md +clickhouse-na-highload-2016.md 2016/clickhouse-na-highload-2016.md +clickhouse-meetup-v-novosibirske-3-aprelya-2017.md 2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017.md +clickhouse-meetup-v-minske-itogi.md 2017/clickhouse-meetup-v-minske-itogi.md +clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md 2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md +clickhouse-meetup-v-ekaterinburge-16-maya-2017.md 2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017.md +clickhouse-na-uwdc-2017.md 2017/clickhouse-na-uwdc-2017.md +clickhouse-meetup-edet-v-minsk.md 2017/clickhouse-meetup-edet-v-minsk.md +clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md 2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md +clickhouse-meetup-v-moskve-5-sentyabrya-2019.md 2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019.md +clickhouse-meetup-v-novosibirske-26-iyunya-2019.md 2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019.md +clickrouse-meetup-v-minske-11-iyulya-2019.md 2019/clickrouse-meetup-v-minske-11-iyulya-2019.md +clickhouse-meetup-v-limassole-7-maya-2019.md 2019/clickhouse-meetup-v-limassole-7-maya-2019.md diff --git a/website/css/blog.css b/website/css/blog.css new file mode 100644 index 00000000000..80ba393dec1 --- /dev/null +++ b/website/css/blog.css @@ -0,0 +1,8 @@ +body.blog .dropdown-item { + color: #111 !important; +} + +body.blog .dropdown-item:hover, +body.blog .dropdown-item:focus { + background-color: #efefef; +} diff --git a/website/locale/en/LC_MESSAGES/messages.po b/website/locale/en/LC_MESSAGES/messages.po index c2c37ebf2b1..9b264bb67d4 100644 --- a/website/locale/en/LC_MESSAGES/messages.po +++ b/website/locale/en/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-20 15:32+0300\n" +"POT-Creation-Date: 2020-06-15 22:29+0300\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language: en\n" @@ -37,27 +37,27 @@ msgstr "ClickHouse - fast open-source OLAP DBMS" msgid "ClickHouse DBMS" msgstr "ClickHouse DBMS" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "open-source" msgstr "open-source" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "relational" msgstr "relational" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytics" msgstr "analytics" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytical" msgstr "analytical" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "Big Data" msgstr "Big Data" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "web-analytics" msgstr "web-analytics" @@ -77,6 +77,14 @@ msgstr "" msgid "Yandex LLC" msgstr "Yandex LLC" +#: templates/blog/content.html:16 +msgid "Published date" +msgstr "Published date" + +#: templates/blog/nav.html:18 +msgid "Documentation" +msgstr "Documentation" + #: templates/docs/footer.html:3 msgid "Rating" msgstr "Rating" diff --git a/website/locale/es/LC_MESSAGES/messages.mo b/website/locale/es/LC_MESSAGES/messages.mo index 9b61bb401bb8d56b03085f60b85d14bdc3c28009..a072272644ef6c4831ba813f62d3c5723d0d5b28 100644 GIT binary patch delta 1299 zcmYMzUr3Wt7{~EPo%yGn`Df;zsDY6oD{y6KrKFfeNr|8#NVw)eF{hyt6>Shf7ZHSz zWaWhgc~ciUib{%+cM$|BK@@pmMd(EeL4PRf`@@$S@6P9(_kGWKo;~MmdHc+c==*Hf zRpWDj?|Qy%i5mU?+;o}cFdRiUPGA*zz3wgQ=J)u;f6umoG2=Mm(x3pAJc z0YkWsv6q9OGgg5bH)m3Rz1YJI?e$sQh(o9ZCb0}3VKXkGwxWhcrEfw74&!DVcJ4pH z4UFHQ*8Ps!(oD)4$5Ql>*kTs-SLyPpe+ssv_No(=z$xUES_HMnw@{@Ucg7Q_(mud# z_!Tw3m=mjn{CFB$QHji;)>}aR0C&{0a`;M70gqq{wxCLU4^{HV$YpUF1$d20=p$<0 z0;;sj*oSVS>5xXTQComoZ=SmAEG?lDi2k6XL*wDk+k}U)9>=f&KO@z%a!yDifyUYSocBFD&-0w;eK*_ovo*I+ z<=i(uU3{Z_drCC={~2-2su^an0`KAme2Qyv5`&n>8hnc!_Jva;E@35>m;N(~dcGO! z&2pCHVha-|J+oNL_#tk>Dcp!N_y~*m3oorPOW}op8HY`A;+DNaB`A3FAFzz^JgTr| zOyE~+Aive|8s$vvM-`IBQ#gR@@f}96h*kI-RbYsy3fP26Y{7MS)|($jYP8EpF1z8) zkD@wy4~NKa&$&>=QTkZxBM_3q4iile_M!^t!)6@9ZhVFvxPk|)PfsK9yD zQohGG_#K-mY=Sncb91a?0PCyizY_0cLD-F!K&|OfRLlCj@o7{``>_RIp#BY?P=zex zFoxKvD&Q6>-V;>e1#G|tR6c*092m@Tq1K*2welP)@F1#?>!=5AqgpwJ=kXJ(?8+KtQ|J5GM-QvEyVL65l#eg{4q? zCXGtei(1p;sDk=Yg=bL(T|>ng!@W4}-Je79`mGpo$DMtCcO_8px%sjVC)mC#5!)Vb W-`$#U|AYog-HX+SO5Bg(tn(K;qjmTI diff --git a/website/locale/es/LC_MESSAGES/messages.po b/website/locale/es/LC_MESSAGES/messages.po index 794662f0dbd..54765875a81 100644 --- a/website/locale/es/LC_MESSAGES/messages.po +++ b/website/locale/es/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-20 15:32+0300\n" +"POT-Creation-Date: 2020-06-15 22:29+0300\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: es\n" @@ -36,27 +36,27 @@ msgstr "ClickHouse - DBMS OLAP de código abierto rápido" msgid "ClickHouse DBMS" msgstr "Sistema abierto." -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "open-source" msgstr "de código abierto" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "relational" msgstr "relacional" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytics" msgstr "analítica" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytical" msgstr "analítico" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "Big Data" msgstr "Grandes Datos" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "web-analytics" msgstr "Sistema abierto." @@ -76,6 +76,14 @@ msgstr "" msgid "Yandex LLC" msgstr "Sistema abierto." +#: templates/blog/content.html:16 +msgid "Published date" +msgstr "Fecha de publicación" + +#: templates/blog/nav.html:18 +msgid "Documentation" +msgstr "Documentación" + #: templates/docs/footer.html:3 msgid "Rating" msgstr "Clasificación" diff --git a/website/locale/fa/LC_MESSAGES/messages.mo b/website/locale/fa/LC_MESSAGES/messages.mo index 36d0063da7a33529fbb93d0dc770594a8f758d80..21ba91517111062c683f8c18545bc2846144588e 100644 GIT binary patch delta 1296 zcmYM!TS!zv9LMp$x_Q6kddp1OG_t_3mJ;vK&58_yKuM(XmPw^XD^Z|pH+|_rgf3=2 zmLXEu21UMTy+ja3_;5rkBIrS(0*j>R^8Lv|huQO)nRCv}fBrMRE%8fS0?enN`<<(7l4Wo8MO8?*%VeFbKl`Ry>3 zm2_MTI*1vxpI|CZVKUBRKU%oi54eTfurI=lVpH7Wa29#h=1>!TMT&(IZ4^eM0!&6P z^IJNVwRBWqES^RM)QR=jkIV5lrei##xN$uy&|=hpRak~K7=r`B{vlN6Mv$0o3>DBb zJ!gJ}6lyKn>EKm{;?dH4eL;Ua3FSoTBhEZl_oxEh<#gFUzo z@1eGG67~KZx(VnH-oZpS`LE%@W1cF-$(%IpRVMO|<)Q*C4fa=}R@{hszYWXr9B#pJ zR3P6`0i+P^D3)L*e#R_}WwmPO`Kf5d`%x=Ci$3f`rFa1~k)oacGYVHdYOYE4ZC zeRqscC*LsNJ$dT=|D5-kmC{XOG2X%z_!yVsFa|J#tMN5**#~a5_z6p}aLGTzsQ05- zZI-e&8fzFh=9*98W1oq=f`dJQ) zGW~(Li zZlh#2gj%SFgQ9t^q5|m0-RR(5T*MQ2n=+lo)B+9d`AKF^;#pKc^Qek=`BTst38N}g ziCUl$73fwZY1@T514mGYtrv;W5~u{O;{m*fs=yplIj_x!ork_1US~Fc%;RJVT6}@# bhFDW%Q*-O)n6p?E@jKD7={%>Z{HXsg=ly(S diff --git a/website/locale/fa/LC_MESSAGES/messages.po b/website/locale/fa/LC_MESSAGES/messages.po index 65eeb6f605a..cc50b4726ff 100644 --- a/website/locale/fa/LC_MESSAGES/messages.po +++ b/website/locale/fa/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-20 15:32+0300\n" +"POT-Creation-Date: 2020-06-15 22:29+0300\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: fa\n" @@ -36,27 +36,27 @@ msgstr "ClickHouse - سریع باز-منبع OLAP DBMS" msgid "ClickHouse DBMS" msgstr "خانه عروسکی" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "open-source" msgstr "متن باز" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "relational" msgstr "رابطه" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytics" msgstr "تجزیه و تحلیل" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytical" msgstr "تحلیلی" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "Big Data" msgstr "داده های بزرگ" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "web-analytics" msgstr "تجزیه و تحلیل وب سایت" @@ -76,6 +76,14 @@ msgstr "" msgid "Yandex LLC" msgstr "Yandex, LLC" +#: templates/blog/content.html:16 +msgid "Published date" +msgstr "تاریخ انتشار" + +#: templates/blog/nav.html:18 +msgid "Documentation" +msgstr "مستندات" + #: templates/docs/footer.html:3 msgid "Rating" msgstr "درجهبندی" diff --git a/website/locale/fr/LC_MESSAGES/messages.mo b/website/locale/fr/LC_MESSAGES/messages.mo index 6d5d70236c139ece8255fb2ceaee84ce6c7d96b3..6ca1195547545385bed08d88ae6b5c5682b839fd 100644 GIT binary patch delta 1295 zcmZA0OGs2v9LMp$dVSVtKJwXPnjRL6i$)F26eGnT3qvpj6(?sx(X>!27da6@8;dZC zwA|VvH`5>(M%ZH!Eh@q`6%_<7YLT0Yf~fBgF9Z#D?&q8{_x#WQ{?C~=fqSKq_gT(0 zizrNa?G;nPGLGmF%uUs1z%wrzQuf8LN5Emtr(p|vkdgR?m^9O#LZ?A z>!4A{zlFWX zT=tNg5_ye-xQt3T#7S0Sr%-XPMriOiuo=`=yu@}~M3p{3y|fiINDQk-jklsI6h3V0_r;_OTC)D3hE z9}o8r1&4YD`eXCSkK@vthL86340eY@-cWET96Os@>7?x{*;(!l1a=21V+)>5iT_Re Zm{k~`UB|=TP}qAy)4NvU{mcnEe*r*3iE;n{ delta 1200 zcmYMyUr3Wt7{~Eve{`vv)^yf%MaxK}EpjG^lq6=05L!xsgphSpL^q{De{6K&MgJ}g zioA=Upf0-b^`c?jSQz3(HxgKqT2Zi|pe}+%*7t`mG|t}7IcIyH^PK0LT+`cB_I=oY z+xYC|8{^yM)#(4{lHaV7;SiSN2(H0CVq$3FCQGVxKXE zUvV?*+Xl7~WTFF=NCuDNDO`=OF^Y>A#sVtA5K$Gd0o!mJmf>l4{ydVST}IZjVR!x} zs-m}WkoD~$9c3J&j3F^c-V9ch;4af1yU!u5>fb=;AQK975f%6f((I9I4ppIP+=TC3mr;S^oRco^PSm&;wZH(r z#Ot`5#4-fg!+3;q)JEQfslPH>VnVk!k1Dy315(M#P-h&$9aw`~@Bq^691ksc4bS2O zRH9LCtP\n" "Language: fr\n" @@ -36,27 +36,27 @@ msgstr "ClickHouse-SGBD OLAP open-source rapide" msgid "ClickHouse DBMS" msgstr "SGBD ClickHouse" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "open-source" msgstr "open-source" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "relational" msgstr "relationnel" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytics" msgstr "Analytics" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytical" msgstr "analytique" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "Big Data" msgstr "Big Data" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "web-analytics" msgstr "web-analytics" @@ -76,6 +76,14 @@ msgstr "" msgid "Yandex LLC" msgstr "Yandex LLC" +#: templates/blog/content.html:16 +msgid "Published date" +msgstr "Date de publication" + +#: templates/blog/nav.html:18 +msgid "Documentation" +msgstr "Documentation" + #: templates/docs/footer.html:3 msgid "Rating" msgstr "Évaluation" diff --git a/website/locale/ja/LC_MESSAGES/messages.mo b/website/locale/ja/LC_MESSAGES/messages.mo index 4577e84e199bbcf81da749052a34b8ce358e4e1f..4f72203e20bd51ad0351a33205ed17e01d4d4072 100644 GIT binary patch delta 1279 zcmYMzdq`7J9Ki9PdVTB6`N*l%%xEGCwjO9LnNeA^KQg2TNM$M|nNm&yAs4Z-D5xy0 zW&YDc352jw2oi|}QGXPOMW9p=4J@LFKnS9~KReJkyPxy8_nhDPoqJ!exV19y&TjPz zzs>yf`PW5j>;LbDC6YmN0Btye={SxFIEhI(jX4-Wiu~c?LMv7z4NHt3)aQLzC=!r8 zRC4J!X&k^j+M}3_Q<#ZgFoY5(G6&bO3VY&3C^E$*6=#rJFUPKM#8kXRFtj70PjxKhw5%-}U%^wu3Aop3J}VGC+ty|@WOs2Tczy7Qm72yL9pGAzXMD0Ihm?w;Q{YKokgg~g`dYx=8Ay9T|C??MlTP~V9+ zs9(bmJb}r)!5a8^)JzScZs28ribnbsb>SRVtHs5rnen3r(rvtidS5?k#%`MSJ=FUj z;%$766lvuXDr2WX)cYSS{C5Nss4snBlFAk;b9ex&2-}T!u>xmM&p4a#Yp?({fOgct zI#G`*X!_5X_60M3**Jh2(2!|ALINt3XH;~;DJ;Wj(@xtL@44qb2RBVsCO-y! zw~gO+{=)q2&QSFKciv}~PuPQbcoUc4eO!pcn1iFZ3|}G5K5?nQFBrhw%z4A8_oG;5 zmb5xL%Nb~Q_Fysb9bAedSb!6F7pL(Do?T#;z%yB9G#lZv0jDbmQGfj&=cA%dWWb zK2$~fv5WQX5glb5rjErPCPLP*UCgEz526xi#VB@R6F$HioW({gV;4G#t*pFo;5ap`3PEvQPR`CxSR zbI!l0@c@IGuQWjYSJT?TuoR0ppA}e(Yq1UW-c_X8 z0GC_%80$$aL75K_Um^=_cq&N!Rhl#dg*fAG{6>v?nN1J+u^tO>3$~!Xfqv9i{{%1M zI4a>Ce6gxfJ1X((s04>l^G#wien`?`EsL?b7HmL0um@G5W)~kpJ$DQr;yI*QgnyA3 zJF7-L-%&JQ@=h!#zKlC@2s`jQ#<7*GH)HY!9i8cG+>h^239MurN+^mtnrhT|t&0=x zexq|QDxt$JK7~rC8}(i<#<0)DFOXC8+RISto^P8sHJdf&NsVUL_;RY&#bc3;RkfAz T)E|F1GZo1{osk+Syyp7{`FwRL diff --git a/website/locale/ja/LC_MESSAGES/messages.po b/website/locale/ja/LC_MESSAGES/messages.po index c88f2cabea2..f057f359724 100644 --- a/website/locale/ja/LC_MESSAGES/messages.po +++ b/website/locale/ja/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-20 15:32+0300\n" +"POT-Creation-Date: 2020-06-15 22:29+0300\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: ja\n" @@ -33,27 +33,27 @@ msgstr "ﾂ環板篠ｮﾂ嘉ｯﾂ偲青エﾂδﾂ-ﾂエﾂスﾂ-ﾂシﾂ" msgid "ClickHouse DBMS" msgstr "クリックハウスDBMS" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "open-source" msgstr "オープンソース" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "relational" msgstr "関係" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytics" msgstr "analytics" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytical" msgstr "分析" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "Big Data" msgstr "ビッグデータ" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "web-analytics" msgstr "ウェブ分析" @@ -71,6 +71,14 @@ msgstr "ソフトウェアは、明示または黙示を問わず、いかなる msgid "Yandex LLC" msgstr "Yandex LLC" +#: templates/blog/content.html:16 +msgid "Published date" +msgstr "公開日" + +#: templates/blog/nav.html:18 +msgid "Documentation" +msgstr "文書" + #: templates/docs/footer.html:3 msgid "Rating" msgstr "評価" diff --git a/website/locale/messages.pot b/website/locale/messages.pot index 75aae739cff..12cb2a98bb8 100644 --- a/website/locale/messages.pot +++ b/website/locale/messages.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-20 15:32+0300\n" +"POT-Creation-Date: 2020-06-15 22:29+0300\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -32,27 +32,27 @@ msgstr "" msgid "ClickHouse DBMS" msgstr "" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "open-source" msgstr "" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "relational" msgstr "" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytics" msgstr "" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytical" msgstr "" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "Big Data" msgstr "" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "web-analytics" msgstr "" @@ -70,6 +70,14 @@ msgstr "" msgid "Yandex LLC" msgstr "" +#: templates/blog/content.html:16 +msgid "Published date" +msgstr "" + +#: templates/blog/nav.html:18 +msgid "Documentation" +msgstr "" + #: templates/docs/footer.html:3 msgid "Rating" msgstr "" diff --git a/website/locale/ru/LC_MESSAGES/messages.mo b/website/locale/ru/LC_MESSAGES/messages.mo index 38fc03212af92bef1bc9b77364958447cf048c5d..53d8b37d07246761c4600def95d5998e8cce20ee 100644 GIT binary patch delta 1296 zcmYMzT}YEr7{KvIH=SD3{LH20%5X+Tv@rZ?k|qUt5eZ3TAEts@DbfagkOK=N=%UO> zEe*QJo1)n+GIxVoRV0<_ASU7nCgV89;S46=JZ9n#&Z*M`rslYUZ9JF-r(F zpm*BO`tp&2rg#c{_#HL!6XZ23O5{AUw%lS?9e5WtkUlKHS9k~)un1l3q6*KV9?dh{ zk0WM%8CgPpq2EHmO5QYealG{SGf*RSAajb-v;Z}=MaWq3pe}F(U3lE=??fKC$!nGk z^x$6Vty~1n*bwI7SQ7cyhHo@z2G+0ylgYPET#dC@i`xIvbOJY0pT%-qM&0>t(zY3! zQRnSOy+cni2jAdB0-D2kcG{6j{?E{HEj@C^^QgCY3AbVlqf60+dI=klwd6ADncl`$ z>_ZLILR32bByzjOi=VU~_21P_X9eEDojBz;8-Amv+{3}@9jHQGD1@5oNz|Rqp)R~? z8p}=9%b1QkFdvuj7;47KNkbkxtwCaxt4yNv-NQok4^cQpA&gJZy(N;78T?nziKKc4 zd+A?9<`%y*vhV=voq2)mK>}v|18QKCs2f?pd|W}jd=Apf@3%=V1&yo>_3u`Ly2IDV zCgpwbinY>WKiJ;jyV%s`t@pOHw*^DCiKxV*zH=8^E;TnbIveY~O~LE&#nyy`1 z-R=^1ad6zeVvFj%?{Q`Z1L6aLA`mwyLQF8aDT<))j~!^7ozMHc=bZO_-sik07hUR!&sVx{ z8^8Vh>-ZlkRP_Hh>NZ@|ba0iW8UY1!{s>r$2*U;yF}eU$Fzf z;YQ}S0Lv(&BZ^AoFrLFfT#4_{k2$Qw->3wAjH&@!FoIjK6fZjcBS?;R1)0mPIsG?K z6&=SI^V>5T%D9d?))p`jGKU>xG(C6}mB0yX!x;AB6YRuA+=or9La*XH?!ik=oIs|q zm*~Pb=)*L|^}0XNP^NRpSoYa*8CB9h$YBLMr3uQh8U3j1-N<3RoIbLEJ`54J^Ws&h zGZ@4%Cw_{mz^e-C-$f%shX$^o-p8;8b;FS3bzDn)5BFjMwelZWkFAtX6CFV9nE`CT zQM^M!_tDErw=?J|`Xg2OSNouf`md*BijH0Q9<|wgjKbKq0k!KxcmaD*2|h>d_9C)k z7h3>7=sM~v@^i5VcVH`yIPoN^(hC^FCH!EnpBg;V5deO``6fLM8SV^;Kn23ye~B)=+3ik`wNzE4k>IEl5s#JKZJW c_KvNM+rnL;j^tn8cTcjoI^#@~ diff --git a/website/locale/ru/LC_MESSAGES/messages.po b/website/locale/ru/LC_MESSAGES/messages.po index aed233c1275..981b83f3d7e 100644 --- a/website/locale/ru/LC_MESSAGES/messages.po +++ b/website/locale/ru/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-05-20 15:32+0300\n" +"POT-Creation-Date: 2020-06-15 22:29+0300\n" "PO-Revision-Date: 2020-03-26 10:19+0300\n" "Last-Translator: FULL NAME \n" "Language: ru\n" @@ -38,27 +38,27 @@ msgstr "ClickHouse-быстрая СУБД OLAP с открытым исходн msgid "ClickHouse DBMS" msgstr "СУБД ClickHouse" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "open-source" msgstr "открытый исходный код" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "relational" msgstr "реляционный" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytics" msgstr "аналитика" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytical" msgstr "аналитический" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "Big Data" msgstr "большие данные" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "web-analytics" msgstr "веб-аналитика" @@ -78,6 +78,14 @@ msgstr "" msgid "Yandex LLC" msgstr "ООО «Яндекс»" +#: templates/blog/content.html:16 +msgid "Published date" +msgstr "Дата публикации" + +#: templates/blog/nav.html:18 +msgid "Documentation" +msgstr "Документация" + #: templates/docs/footer.html:3 msgid "Rating" msgstr "Рейтинг" diff --git a/website/locale/tr/LC_MESSAGES/messages.mo b/website/locale/tr/LC_MESSAGES/messages.mo index 1cfc686a5ed1392629766d00ad23bfdb739bd789..65218c3b7298969fc73112ec30893466695a3a4b 100644 GIT binary patch delta 1296 zcmYMzTS!zv9LMp$dc5X6v$hRdGO@xQ+)^}6P%Nd4iog&g+;maNOL=+d=0b!)PbCCd zN?>F!32qBv2qNZ75f{3)vYe zTj{vwI)Yx>lbDSQn1$bQ2rbTR4entD4#t~NY=K7#E+T)mWz<3|NU><5O~fQrfLZ8g zee+P+Nk=Uv;{{Ye{n&&JoV9__I-wdXPEI=bDd7Z80*|%25j(#XPKYfA2?%UFR{t1_n{{ zc(@3gFc&vtX&U+0M5pM`f{nNXTio_d%%^=DDK_mokIK*z`fvp`E`xK@(e20WcpP0|$bSVLO&q9F`v|qean}fPOYJ#oryo$M`-(dApQzNXVG$N`j>KZ^ zn2T4i2S<@&+1z9Wwg(l^kq{L{+=xp3Wju-fs1&|Mo&6H(vVB1X7{h2~ED;r0I%+~M zkMmgIwnwpyolc@I;VS9SMji66*FBU>rH0CGtjBIF!Pi)YalF*)XN-qw*P#L$K?N{| z3Va+D*aT{UXQ;b1jXL{T)Q09!N51&4KgYgN(E_XJM_p}Av=5ca5>MokQyG&|)z#G3 z5$x;@^tN?%MyBFlM5UhSJKNsY(;95{H3xcwk*kR%PEyg{fq~9cZ91FHibb(ZN~}yef11>25yXPX;-ZP&5Lqmkgs8>^7m{5} zVqtDfj6@=4Z|^oYyCCdBLT|{z5`vUO;-(29zQ1@A&FT9&=QQtmp7Wg3=dog2?p?^a zZhTUF!+iIYY4rbd!7;04n8hj_#07X8=ixY3-~=|{Q>58@9xeD0tFdyVvsl9ObIp=?Dqu#lJP&@dNmPI-Z@z%#jAu}ZeZm-i z#TCT2rEH^;iB41^`>_X4U=6;+Mx4bE{zN4hWK}KLiV0kc^Rd^PKZoRK7m-+Y*_$6g zRrDJ66W{L9QO05F*yLj&B!=x~HNAKMmB0~$mC_9pgXl<`5N*(J|GRE0*c8C}$S1=Nw1a2bX;FWs3KD)25G z!;HoxR>LAm#;Z6-RrXkj`fCS$OmG|RENX|tsFIDM&iE0kqknZJM^Nh&kP7\n" "Language: tr\n" @@ -36,27 +36,27 @@ msgstr "ClickHouse - hızlı açık kaynak OLAP DBMS" msgid "ClickHouse DBMS" msgstr "ClickHouse DBMS" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "open-source" msgstr "açık kaynak" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "relational" msgstr "ilişkisel" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytics" msgstr "analiz" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytical" msgstr "analitik" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "Big Data" msgstr "Büyük Veri" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "web-analytics" msgstr "web-analyt -ics" @@ -76,6 +76,14 @@ msgstr "" msgid "Yandex LLC" msgstr "Yandex LLC" +#: templates/blog/content.html:16 +msgid "Published date" +msgstr "Yayım datelandığı tarih" + +#: templates/blog/nav.html:18 +msgid "Documentation" +msgstr "Belge" + #: templates/docs/footer.html:3 msgid "Rating" msgstr "Verim" diff --git a/website/locale/zh/LC_MESSAGES/messages.mo b/website/locale/zh/LC_MESSAGES/messages.mo index 7260ce95e89b510777ffebc6c96cb915de5e9939..0a8eabb78ba784581b73d2e56aa0591383c04889 100644 GIT binary patch delta 1281 zcmYMzT}V@57{KvIJ^3-ea#L%5#L#d;Q)rcGmJk+2Ds&Om$TCww``Cya;YAKC5-%i^ zvP_3p8HfZnDq%OHE~IXPu&bhwazq6NQi726f9i$C+4;TC`}v&rJ!i6Vr84y1Zrv7s z2l*HAZ%)+c|8Lw9aS(~ z>x+}ZW-1yn123W`)QfF6itF(gIx&k`>{yPPXf5i5$FL5YF&)QD{R7mUn?`b$C~88_ zw4L?kB?aB$R~W>PsG0lN*TMvmbI97#࿒sM$r=)p&L06$G9{IX6i)d6qnJ1y0z8FSgAu@paI=@)U@{^mt5m^od+03 z?eFA8P|Hx?iEY{Jzh+WTqzg8oKG_iy`%xzjn))yn6F)V+F)pJXVGie1f>oG@KGgmJ z+=PSJgVXp8%W~NNofI4#sAkrHn#gfvPH90d37Gmm)P4gd9>FcdVbpiw1?sp()WmNy`T_h& zJY?GY$wJ59Lfz@m7zKTT$9M=AuogXhdzwfm>KRX%cnWob1>~(akshl)DRW<-EqI~B-`(2X74S!*DbEwK4hPSi z@4D34(e7$*?e2(NNvpAJRTW!zxV+w7-kL}(^MsWgAL;)TeG)0KKTOD2ym51J_G3+^hp{R+w?vG$(G^D^!75mtVk2;v%ZCub9AZ z*g$=&<2C#w(x^sy@Dg6b8l1-nE@KFPp&AS@t0vr!DU9P5yzcV-NRM_Ksb#}1e-E{y z53rB=Ho=8DjOXcm~x#FE(QzcHuZCv4ovi&s%6KuHsR=<>D!%guO;D zzQq6*FsJQa;zFGkk-2Qe`2)42zmdZ{j8cIhHev*IzXLg}i_=|R;4JF-B0m_piuw(# zp&BVCP`#RJ)?aTHc8M0$#7R{C0`A5E=cw}qHjrPyz4#NiV~qXM^9iiO9_+^(_#IdA z5H7OZYAhIH{TXXfHdcGv#7Ui{P!Dvv_%t>VXHk1Uj2ib8)$nu7;u5NnRyJ7oPoU<> zqViWz^ABP>-pO&HC3%V3n>qKuJLkM}5jD|A?8cJIC;4lv5g$X{Z>Ne1q){uFK{b9E zGdO}r@B^xWT$ulu_Vg4gkwr}~;(UZ^?6Hf-UHlAHa1u9Oz}=s77F_x5#jGbkQ\n" "Language: zh\n" @@ -33,27 +33,27 @@ msgstr "ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人" msgid "ClickHouse DBMS" msgstr "ﾂ环板msｮﾂ嘉ｯﾂ偲" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "open-source" msgstr "开源" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "relational" msgstr "关系" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytics" msgstr "分析" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "analytical" msgstr "分析" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "Big Data" msgstr "大数据" -#: templates/common_meta.html:24 +#: templates/common_meta.html:28 msgid "web-analytics" msgstr "网络分析" @@ -71,6 +71,14 @@ msgstr "软件按\"原样\"分发，不附带任何明示或暗示的担保或 msgid "Yandex LLC" msgstr "Yandex LLC" +#: templates/blog/content.html:16 +msgid "Published date" +msgstr "发布日期" + +#: templates/blog/nav.html:18 +msgid "Documentation" +msgstr "文件" + #: templates/docs/footer.html:3 msgid "Rating" msgstr "评分" diff --git a/website/main.html b/website/main.html index 73693204e5d..95debc2a233 100644 --- a/website/main.html +++ b/website/main.html @@ -3,6 +3,7 @@ {% set language = config.theme.language %} {% set direction = config.theme.direction %} {% set is_amp = config.extra.is_amp %} +{% set is_blog = config.extra.is_blog %} {% set single_page = config.extra.single_page %} {% set no_footer = True %} {% set og_type = 'article' %} @@ -36,22 +37,38 @@ {% if is_amp %} {% include "templates/docs/amp.html" %} {% else %} - {% set extra_html_attrs = 'data-version="' + data_version + '" data-single-page="' + data_single_page + '"' %} - {% set extra_body_attrs = 'data-spy="scroll" data-target="#toc" data-offset="80"' %} - + {% if not is_blog %} + {% set extra_html_attrs = 'data-version="' + data_version + '" data-single-page="' + data_single_page + '"' %} + {% set extra_body_attrs = 'data-spy="scroll" data-target="#toc" data-offset="80"' %} + {% else %} + {% set extra_body_attrs = 'class="blog"' %} + {% endif %} {% extends "templates/base.html" %} {% block content %} - {% include "templates/docs/nav.html" %} -
-
- {% include "templates/docs/sidebar.html" %} - {% include "templates/docs/content.html" %} - {% if not config.extra.single_page %} - {% include "templates/docs/toc.html" %} - {% endif %} + {% if not is_blog %} + {% include "templates/docs/nav.html" %} +
+
+ {% include "templates/docs/sidebar.html" %} + {% include "templates/docs/content.html" %} + {% if not config.extra.single_page %} + {% include "templates/docs/toc.html" %} + {% endif %} +
-
+ {% else %} + {% include "templates/blog/nav.html" %} +
+
+ {% include "templates/blog/content.html" %} +
+
+ {% if page and page.meta.is_index %} + {% include "templates/index/community.html" %} + {% include "templates/blog/footer.html" %} + {% endif %} + {% endif %} {% endblock %} {% endif %} diff --git a/website/sitemap-index.xml b/website/sitemap-index.xml index e53d6c29c54..75fdc75973c 100644 --- a/website/sitemap-index.xml +++ b/website/sitemap-index.xml @@ -21,6 +21,12 @@ https://clickhouse.tech/docs/fa/sitemap.xml + + https://clickhouse.tech/blog/en/sitemap.xml + + + https://clickhouse.tech/blog/ru/sitemap.xml + https://clickhouse.tech/sitemap-static.xml diff --git a/website/templates/blog/content.html b/website/templates/blog/content.html new file mode 100644 index 00000000000..38ad7933b00 --- /dev/null +++ b/website/templates/blog/content.html @@ -0,0 +1,43 @@ +
+ {% if not page.meta.is_index %} +
+
+ {% if page.meta.image %} + {{ title }} + {% endif %} +

{{ title }}

+
+ +
+ {{ page.content|adjust_markdown_html }} +
+ +
+ {{ page.meta.date }} + {% if page.meta.tags %} + {% for tag in page.meta.tags %} +
+ {{ tag }} +
+ {% endfor %} + {% endif %} +
+ {% include "templates/blog/footer.html" %} +
+ {% else %} + {% for post in config.extra.post_meta.values() %} + + {% set post_image = post.get('image') or '/images/index/intro.svg' %} +
+
+ {{ post['title'] }} +
+
+

{{ post['title'] }}

+ {{ post['date'] }} +
+
+ + {% endfor %} + {% endif %} +
diff --git a/website/templates/blog/footer.html b/website/templates/blog/footer.html new file mode 100644 index 00000000000..3e94ecce51f --- /dev/null +++ b/website/templates/blog/footer.html @@ -0,0 +1,9 @@ +
+
+
+
+
{{ config.copyright }}
+
+
+
+
diff --git a/website/templates/blog/nav.html b/website/templates/blog/nav.html new file mode 100644 index 00000000000..a7e135296f2 --- /dev/null +++ b/website/templates/blog/nav.html @@ -0,0 +1,45 @@ + diff --git a/website/templates/common_meta.html b/website/templates/common_meta.html index 84bd93d5175..86a852284ee 100644 --- a/website/templates/common_meta.html +++ b/website/templates/common_meta.html @@ -10,7 +10,11 @@ +{% if page and page.meta.image %} + +{% else %} +{% endif %} {% if page and not single_page %} @@ -20,13 +24,18 @@ {% include "templates/docs/ld_json.html" %} +{% if page and page.meta.tags %} + +{% else %} +{% endif %} {% if config and (config.extra.single_page or config.extra.version_prefix) %} {% endif %} -{% if config and page %} +{% if config and page and not is_blog %} {% for code, name in config.extra.languages.items() %} {% endfor %} diff --git a/website/templates/docs/ld_json.html b/website/templates/docs/ld_json.html index 3db89657221..7170a88dad0 100644 --- a/website/templates/docs/ld_json.html +++ b/website/templates/docs/ld_json.html @@ -1,12 +1,17 @@ {% if page and page.meta %} +}{% endif %}] {% endif %} diff --git a/website/templates/index/community.html b/website/templates/index/community.html index e230cac8da9..0adb3150ea0 100644 --- a/website/templates/index/community.html +++ b/website/templates/index/community.html @@ -113,8 +113,8 @@ class="bg-secondary-alt rounded-circle p-2 mr-4 float-left" />
{{ _('ClickHouse Blog') }}
-

{{ _('in') }} {{ _('English') }} - or in {{ _('Russian') }}

+

{{ _('in') }} {{ _('English') }} + or in {{ _('Russian') }}

diff --git a/website/templates/index/nav.html b/website/templates/index/nav.html index e3c680f1885..9bae81eb73c 100644 --- a/website/templates/index/nav.html +++ b/website/templates/index/nav.html @@ -18,8 +18,7 @@ Documentation
  • - Blog + Blog
  • Community From 66c0ef26104f9d135e0f5f5413b01fdc575868ad Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 16 Jun 2020 11:08:32 +0300 Subject: [PATCH 268/329] Do not materialize header for merge tables. --- src/Storages/StorageMerge.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 64d0f11f853..1584acbb9d4 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -407,7 +407,6 @@ Block StorageMerge::getQueryHeader( if (query_info.prewhere_info) { query_info.prewhere_info->prewhere_actions->execute(header); - header = materializeBlock(header); if (query_info.prewhere_info->remove_prewhere_column) header.erase(query_info.prewhere_info->prewhere_column_name); } @@ -415,9 +414,9 @@ Block StorageMerge::getQueryHeader( } case QueryProcessingStage::WithMergeableState: case QueryProcessingStage::Complete: - return materializeBlock(InterpreterSelectQuery( + return InterpreterSelectQuery( query_info.query, context, std::make_shared(getSampleBlockForColumns(column_names)), - SelectQueryOptions(processed_stage).analyze()).getSampleBlock()); + SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); } throw Exception("Logical Error: unknown processed stage.", ErrorCodes::LOGICAL_ERROR); } From 86bafbe6c342f43281e2b03c0f009d24881397d4 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 16 Jun 2020 11:25:59 +0300 Subject: [PATCH 269/329] Added test from #9043 --- .../01319_mv_constants_bug.reference | 0 .../0_stateless/01319_mv_constants_bug.sql | 23 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/queries/0_stateless/01319_mv_constants_bug.reference create mode 100644 tests/queries/0_stateless/01319_mv_constants_bug.sql diff --git a/tests/queries/0_stateless/01319_mv_constants_bug.reference b/tests/queries/0_stateless/01319_mv_constants_bug.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01319_mv_constants_bug.sql b/tests/queries/0_stateless/01319_mv_constants_bug.sql new file mode 100644 index 00000000000..975a33d7b71 --- /dev/null +++ b/tests/queries/0_stateless/01319_mv_constants_bug.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS distributed_table_merged; +DROP TABLE IF EXISTS distributed_table_1; +DROP TABLE IF EXISTS distributed_table_2; +DROP TABLE IF EXISTS local_table_1; +DROP TABLE IF EXISTS local_table_2; + +CREATE TABLE local_table_1 (id String) ENGINE = MergeTree ORDER BY (id); +CREATE TABLE local_table_2(id String) ENGINE = MergeTree ORDER BY (id); + +CREATE TABLE local_table_merged (id String) ENGINE = Merge('default', 'local_table_1|local_table_2'); + +CREATE TABLE distributed_table_1 (id String) ENGINE = Distributed(test_shard_localhost, default, local_table_1); +CREATE TABLE distributed_table_2 (id String) ENGINE = Distributed(test_shard_localhost, default, local_table_2); + +CREATE TABLE distributed_table_merged (id String) ENGINE = Merge('default', 'distributed_table_1|distributed_table_2'); + +SELECT 1 FROM distributed_table_merged; + +DROP TABLE IF EXISTS distributed_table_merged; +DROP TABLE IF EXISTS distributed_table_1; +DROP TABLE IF EXISTS distributed_table_2; +DROP TABLE IF EXISTS local_table_1; +DROP TABLE IF EXISTS local_table_2; From 605c904fc64a6d1b8c98c62d1d5f5e518906f9f3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 16 Jun 2020 11:39:12 +0300 Subject: [PATCH 270/329] Add method for compatibility --- src/Storages/MergeTree/MergeTreeData.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 007c6898e60..3fe222ba0b2 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -511,6 +512,13 @@ public: broken_part_callback(name); } + /// TODO (alesap) Duplicate method required for compatibility. + /// Must be removed. + static ASTPtr extractKeyExpressionList(const ASTPtr & node) + { + return DB::extractKeyExpressionList(node); + } + /// Check that the part is not broken and calculate the checksums for it if they are not present. MutableDataPartPtr loadPartAndFixMetadata(const VolumePtr & volume, const String & relative_path) const; From 264e44ecefbb195f1ffb3391297cb77cfaead49c Mon Sep 17 00:00:00 2001 From: franklee Date: Mon, 8 Jun 2020 14:25:01 +0800 Subject: [PATCH 271/329] support distributed DDLs on cross replication clusters. (cherry picked from commit fd09994d91ba4cce7bf5cd76414ec3677aa1b3ee) --- src/Interpreters/DDLWorker.cpp | 26 +++++-- src/Interpreters/DDLWorker.h | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 8 +-- .../__init__.py | 0 .../configs/remote_servers.xml | 45 ++++++++++++ .../test.py | 70 +++++++++++++++++++ 6 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 tests/integration/test_distributed_ddl_on_cross_replication/__init__.py create mode 100644 tests/integration/test_distributed_ddl_on_cross_replication/configs/remote_servers.xml create mode 100644 tests/integration/test_distributed_ddl_on_cross_replication/test.py diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 9f89fa9199b..cdef2f808ae 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -481,6 +481,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) const auto & shards = task.cluster->getShardsAddresses(); bool found_exact_match = false; + String default_database; for (size_t shard_num = 0; shard_num < shards.size(); ++shard_num) { for (size_t replica_num = 0; replica_num < shards[shard_num].size(); ++replica_num) @@ -491,14 +492,30 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) { if (found_exact_match) { - throw Exception("There are two exactly the same ClickHouse instances " + address.readableString() - + " in cluster " + task.cluster_name, ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + if(default_database == address.default_database) + { + throw Exception( + "There are two exactly the same Snowball instances " + address.readableString() + " in cluster " + + task.cluster_name, ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + } + else ///circular replication is used. + { + is_circular_replicated = true; + auto query_with_table = dynamic_cast(task.query.get()); + if(query_with_table == nullptr || query_with_table->database.empty()) + { + throw Exception( + "For a distributed DDL on circular replicated cluster its table name must be qualified by database name." + ,ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + } + if(default_database == query_with_table->database) return; + } } - found_exact_match = true; task.host_shard_num = shard_num; task.host_replica_num = replica_num; task.address_in_cluster = address; + default_database = address.default_database; } } } @@ -621,6 +638,7 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) { try { + is_circular_replicated = false; parseQueryAndResolveHost(task); ASTPtr rewritten_ast = task.query_on_cluster->getRewrittenASTWithoutOnCluster(task.address_in_cluster.default_database); @@ -643,7 +661,7 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) if (storage && query_with_table->as()) checkShardConfig(query_with_table->table, task, storage); - if (storage && taskShouldBeExecutedOnLeader(rewritten_ast, storage)) + if (storage && taskShouldBeExecutedOnLeader(rewritten_ast, storage) && !is_circular_replicated) tryExecuteQueryOnLeaderReplica(task, storage, rewritten_query, task.entry_path, zookeeper); else tryExecuteQuery(rewritten_query, task, task.execution_status); diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index c30f888b5a1..d764eab626f 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -100,6 +100,7 @@ private: void attachToThreadGroup(); private: + bool is_circular_replicated; Context & context; Poco::Logger * log; std::unique_ptr current_context; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 5931bca17ea..8741c49cbbb 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4378,10 +4378,10 @@ void StorageReplicatedMergeTree::sendRequestToLeaderReplica(const ASTPtr & query /// SECONDARY_QUERY here means, that we received query from DDLWorker /// there is no sense to send query to leader, because he will receive it from own DDLWorker - if (query_context.getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) - { - throw Exception("Cannot execute DDL query, because leader was suddenly changed or logical error.", ErrorCodes::LEADERSHIP_CHANGED); - } +// if (query_context.getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) +// { +// throw Exception("Cannot execute DDL query, because leader was suddenly changed or logical error.", ErrorCodes::LEADERSHIP_CHANGED); +// } ReplicatedMergeTreeAddress leader_address(getZooKeeper()->get(zookeeper_path + "/replicas/" + leader + "/host")); diff --git a/tests/integration/test_distributed_ddl_on_cross_replication/__init__.py b/tests/integration/test_distributed_ddl_on_cross_replication/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_distributed_ddl_on_cross_replication/configs/remote_servers.xml b/tests/integration/test_distributed_ddl_on_cross_replication/configs/remote_servers.xml new file mode 100644 index 00000000000..6020f6110df --- /dev/null +++ b/tests/integration/test_distributed_ddl_on_cross_replication/configs/remote_servers.xml @@ -0,0 +1,45 @@ + + + + + true + + node1 + 9000 + replica_1 + + + node2 + 9000 + replica_2 + + + + true + + node2 + 9000 + replica_1 + + + node3 + 9000 + replica_2 + + + + true + + node3 + 9000 + replica_1 + + + node1 + 9000 + replica_2 + + + + + \ No newline at end of file diff --git a/tests/integration/test_distributed_ddl_on_cross_replication/test.py b/tests/integration/test_distributed_ddl_on_cross_replication/test.py new file mode 100644 index 00000000000..544438dec3d --- /dev/null +++ b/tests/integration/test_distributed_ddl_on_cross_replication/test.py @@ -0,0 +1,70 @@ +import time +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, + macros={"shard": 1, "replica": 1, "shard_bk": 3, "replica_bk": 2}) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, + macros={"shard": 2, "replica": 1, "shard_bk": 1, "replica_bk": 2}) +node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, + macros={"shard": 3, "replica": 1, "shard_bk": 2, "replica_bk": 2}) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + node1.query(''' +CREATE DATABASE replica_1 on cluster cross_3shards_2replicas; +CREATE DATABASE replica_2 on cluster cross_3shards_2replicas; + +CREATE TABLE replica_1.replicated_local on cluster cross_3shards_2replicas (part_key Date, id UInt32, shard_id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{replica}') + partition by part_key order by id; +CREATE TABLE replica_1.replicated on cluster cross_3shards_2replicas as replica_1.replicated_local + ENGINE = Distributed(cross_3shards_2replicas, '', replicated_local, shard_id); + +CREATE TABLE replica_2.replicated_local on cluster cross_3shards_2replicas (part_key Date, id UInt32, shard_id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard_bk}/replicated', '{replica_bk}') + partition by part_key order by id; +CREATE TABLE replica_2.replicated on cluster cross_3shards_2replicas as replica_2.replicated_local + ENGINE = Distributed(cross_3shards_2replicas, '', replicated_local, shard_id); + ''') + + to_insert = '''\ +2017-06-16 10 0 +2017-06-17 11 0 +2017-06-16 20 1 +2017-06-17 21 1 +2017-06-16 30 2 +2017-06-17 31 2 +''' + node1.query("INSERT INTO replica_1.replicated FORMAT TSV", stdin=to_insert) + time.sleep(0.5) + + yield cluster + + finally: + cluster.shutdown() + + +def test_alter_ddl(started_cluster): + node1.query('''alter table replica_1.replicated_local on cluster cross_3shards_2replicas update shard_id=shard_id+3 + where part_key='2017-06-16'; + ''') + + node1.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) + assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated where shard_id >= 3 and part_key='2017-06-16'", '3') + + node1.query("alter table replica_1.replicated_local on cluster cross_3shards_2replicas delete where shard_id >=3;") + node1.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) + assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated where shard_id >= 3", '0') + + node2.query("alter table replica_1.replicated_local on cluster cross_3shards_2replicas \ + drop partition toDate('2017-06-17');") + node2.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) + assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated", '0') + From afa666792b840628aaae3cdfb783b5bf29aa693e Mon Sep 17 00:00:00 2001 From: franklee Date: Tue, 9 Jun 2020 10:16:47 +0800 Subject: [PATCH 272/329] typo (cherry picked from commit 76c75c947e4ff240f54ec9f968ea0c3d2ee13551) --- src/Interpreters/DDLWorker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index cdef2f808ae..6e20727bcca 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -495,7 +495,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) if(default_database == address.default_database) { throw Exception( - "There are two exactly the same Snowball instances " + address.readableString() + " in cluster " + "There are two exactly the same ClickHouse instances " + address.readableString() + " in cluster " + task.cluster_name, ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); } else ///circular replication is used. From e26e873b0d48279acb67fc69c8f644ce105b8f65 Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 9 Jun 2020 11:13:27 +0800 Subject: [PATCH 273/329] fix for style check. (cherry picked from commit 0be7b0949029f00cdf9fa48fd0263592704006ce) --- src/Interpreters/DDLWorker.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 6e20727bcca..c7a92f48f3f 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -492,23 +492,25 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) { if (found_exact_match) { - if(default_database == address.default_database) + if (default_database == address.default_database) { throw Exception( "There are two exactly the same ClickHouse instances " + address.readableString() + " in cluster " - + task.cluster_name, ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); - } - else ///circular replication is used. + + task.cluster_name, + ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + } + else ///circular replication is used. { is_circular_replicated = true; auto query_with_table = dynamic_cast(task.query.get()); - if(query_with_table == nullptr || query_with_table->database.empty()) + if (query_with_table == nullptr || query_with_table->database.empty()) { throw Exception( - "For a distributed DDL on circular replicated cluster its table name must be qualified by database name." - ,ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + "For a distributed DDL on circular replicated cluster its table name must be qualified by database name.", + ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); } - if(default_database == query_with_table->database) return; + if (default_database == query_with_table->database) + return; } } found_exact_match = true; From 9fc37e1f75a3e63510881799782b7bad9744a49b Mon Sep 17 00:00:00 2001 From: xPoSx <57260844+xPoSx@users.noreply.github.com> Date: Tue, 16 Jun 2020 12:14:25 +0300 Subject: [PATCH 274/329] Remove functions of other keys in GROUP BY section (#10051) --- src/Core/Settings.h | 1 + src/Interpreters/GroupByFunctionKeysVisitor.h | 115 ++++++++++++++++++ src/Interpreters/SyntaxAnalyzer.cpp | 87 +++++++++++++ tests/performance/removing_group_by_keys.xml | 9 ++ .../01300_group_by_other_keys.reference | 58 +++++++++ .../0_stateless/01300_group_by_other_keys.sql | 30 +++++ ...01300_group_by_other_keys_having.reference | 24 ++++ .../01300_group_by_other_keys_having.sql | 21 ++++ 8 files changed, 345 insertions(+) create mode 100644 src/Interpreters/GroupByFunctionKeysVisitor.h create mode 100644 tests/performance/removing_group_by_keys.xml create mode 100644 tests/queries/0_stateless/01300_group_by_other_keys.reference create mode 100644 tests/queries/0_stateless/01300_group_by_other_keys.sql create mode 100644 tests/queries/0_stateless/01300_group_by_other_keys_having.reference create mode 100644 tests/queries/0_stateless/01300_group_by_other_keys_having.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index adc804c3a28..0693f53db9f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -409,6 +409,7 @@ struct Settings : public SettingsCollection \ M(SettingDateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.", 0) \ \ + M(SettingBool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \ M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \ M(SettingBool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \ M(SettingBool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \ diff --git a/src/Interpreters/GroupByFunctionKeysVisitor.h b/src/Interpreters/GroupByFunctionKeysVisitor.h new file mode 100644 index 00000000000..afcf5a14118 --- /dev/null +++ b/src/Interpreters/GroupByFunctionKeysVisitor.h @@ -0,0 +1,115 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + + +/// recursive traversal and check for optimizeGroupByFunctionKeys +struct KeepFunctionMatcher +{ + struct Data + { + std::unordered_set & key_names_to_keep; + bool & keep_key; + }; + + using Visitor = InDepthNodeVisitor; + + static bool needChildVisit(const ASTPtr & node, const ASTPtr &) + { + return !(node->as()); + } + + static void visit(ASTFunction * function_node, Data & data) + { + if ((function_node->arguments->children).empty()) + { + data.keep_key = true; + return; + } + + if (!data.key_names_to_keep.count(function_node->getColumnName())) + { + Visitor(data).visit(function_node->arguments); + } + } + + static void visit(ASTIdentifier * ident, Data & data) + { + if (!data.key_names_to_keep.count(ident->shortName())) + { + /// if variable of a function is not in GROUP BY keys, this function should not be deleted + data.keep_key = true; + return; + } + } + + static void visit(const ASTPtr & ast, Data & data) + { + if (data.keep_key) + return; + + if (auto * function_node = ast->as()) + { + visit(function_node, data); + } + else if (auto * ident = ast->as()) + { + visit(ident, data); + } + else if (!ast->as()) + { + data.keep_key = true; + } + } +}; + +using KeepFunctionVisitor = InDepthNodeVisitor; + +class GroupByFunctionKeysMatcher +{ +public: + struct Data + { + std::unordered_set & key_names_to_keep; + }; + + static bool needChildVisit(const ASTPtr & node, const ASTPtr &) + { + return !(node->as()); + } + + static void visit(ASTFunction * function_node, Data & data) + { + bool keep_key = false; + KeepFunctionVisitor::Data keep_data{data.key_names_to_keep, keep_key}; + KeepFunctionVisitor(keep_data).visit(function_node->arguments); + + if (!keep_key) + (data.key_names_to_keep).erase(function_node->getColumnName()); + } + + static void visit(const ASTPtr & ast, Data & data) + { + if (auto * function_node = ast->as()) + { + if (!(function_node->arguments->children.empty())) + visit(function_node, data); + } + } +}; + +using GroupByFunctionKeysVisitor = InDepthNodeVisitor; + +} diff --git a/src/Interpreters/SyntaxAnalyzer.cpp b/src/Interpreters/SyntaxAnalyzer.cpp index 4bfae18f9a5..2dc2943d36d 100644 --- a/src/Interpreters/SyntaxAnalyzer.cpp +++ b/src/Interpreters/SyntaxAnalyzer.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -346,6 +347,89 @@ void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_colum appendUnusedGroupByColumn(select_query, source_columns); } +///eliminate functions of other GROUP BY keys +void optimizeGroupByFunctionKeys(ASTSelectQuery * select_query, bool optimize_group_by_function_keys) +{ + if (!optimize_group_by_function_keys) + return; + + if (!select_query->groupBy()) + return; + + auto grp_by = select_query->groupBy(); + auto & group_keys = grp_by->children; + + ASTs modified; ///result + std::unordered_set key_names_to_keep; ///set of keys' short names + + ///check if optimization is needed while building set + bool need_optimization = false; + ///filling set with short names of keys + for (auto & group_key : group_keys) + { + if (!need_optimization && group_key->as()) + need_optimization = true; + + if (auto * group_key_ident = group_key->as()) + { + if (key_names_to_keep.count(group_key_ident->shortName())) + { + ///There may be a collision between different tables having similar variables. + ///Due to the fact that we can't track these conflicts yet, + ///it's better to disable optimization to avoid elimination necessary keys. + need_optimization = false; + break; + } + + key_names_to_keep.insert(group_key_ident->shortName()); + continue; + } + if (auto * group_key_func = group_key->as()) + { + key_names_to_keep.insert(group_key_func->getColumnName()); + continue; + } + else + { + key_names_to_keep.insert(group_key->getColumnName()); + } + } + if (!need_optimization) + return; + + GroupByFunctionKeysVisitor::Data visitor_data{key_names_to_keep}; + GroupByFunctionKeysVisitor(visitor_data).visit(grp_by); + + modified.reserve(group_keys.size()); + + ///filling the result + for (auto & group_key : group_keys) + { + if (auto * group_key_func = group_key->as()) + { + if (key_names_to_keep.count(group_key_func->getColumnName())) + modified.push_back(group_key); + + continue; + } + if (auto * group_key_ident = group_key->as()) + { + if (key_names_to_keep.count(group_key_ident->shortName())) + modified.push_back(group_key); + + continue; + } + else + { + if (key_names_to_keep.count(group_key->getColumnName())) + modified.push_back(group_key); + } + } + + ///modifying the input + grp_by->children = modified; +} + /// Remove duplicate items from ORDER BY. void optimizeOrderBy(const ASTSelectQuery * select_query) { @@ -843,6 +927,9 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( /// GROUP BY injective function elimination. optimizeGroupBy(select_query, source_columns_set, context); + /// GROUP BY functions of other keys elimination. + optimizeGroupByFunctionKeys(select_query, settings.optimize_group_by_function_keys); + /// Remove duplicate items from ORDER BY. optimizeOrderBy(select_query); diff --git a/tests/performance/removing_group_by_keys.xml b/tests/performance/removing_group_by_keys.xml new file mode 100644 index 00000000000..6db641966ec --- /dev/null +++ b/tests/performance/removing_group_by_keys.xml @@ -0,0 +1,9 @@ + + + hits_10m_single + hits_100m_single + + + SELECT avg(length(URL)) FROM hits_10m_single GROUP BY WatchID, CounterID FORMAT Null + SELECT avg(length(URL)) FROM hits_10m_single GROUP BY WatchID, CounterID, WatchID - CounterID FORMAT Null + diff --git a/tests/queries/0_stateless/01300_group_by_other_keys.reference b/tests/queries/0_stateless/01300_group_by_other_keys.reference new file mode 100644 index 00000000000..52a5e7f0002 --- /dev/null +++ b/tests/queries/0_stateless/01300_group_by_other_keys.reference @@ -0,0 +1,58 @@ +6931467.646716369 +6931468.33986355 +6931469.0330107305 +6931469.726157911 +6931470.419305092 +6931471.112452272 +3465734.169931768 +3465734.8630789486 +3465735.5562261306 +3465736.24937331 +3465736.94252049 +3465735.209652544 +3465735.209652544 +3465735.5562261483 +3465735.9027997246 +3465735.902799725 +3465734.516505364 +3465735.209652544 +3465735.209652544 +3465735.9027997246 +3465735.902799725 +3465736.595946905 +3465735.2096525617 +3465735.9027997428 +SELECT max(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n number % 2,\n number % 3,\n ((number % 2) + (number % 3)) % 2\nORDER BY k ASC +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY number % 5\nORDER BY k ASC +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n (number % 2) * (number % 3),\n number % 3\nORDER BY k ASC +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n number % 3,\n number % 2\nORDER BY k ASC +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n (number % 2) % 3,\n number % 2\nORDER BY k ASC +6931467.646716369 +6931468.33986355 +6931469.0330107305 +6931469.726157911 +6931470.419305092 +6931471.112452272 +3465734.169931768 +3465734.8630789486 +3465735.5562261306 +3465736.24937331 +3465736.94252049 +3465735.209652544 +3465735.209652544 +3465735.5562261483 +3465735.9027997246 +3465735.902799725 +3465734.516505364 +3465735.209652544 +3465735.209652544 +3465735.9027997246 +3465735.902799725 +3465736.595946905 +3465735.2096525617 +3465735.9027997428 +SELECT max(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n number % 2,\n number % 3,\n ((number % 2) + (number % 3)) % 2\nORDER BY k ASC +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n number % 5,\n (number % 5) * (number % 5)\nORDER BY k ASC +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n (number % 2) * (number % 3),\n number % 3\nORDER BY k ASC +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n (number % 2) * (number % 3),\n number % 3,\n number % 2\nORDER BY k ASC +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n (number % 2) % 3,\n number % 2\nORDER BY k ASC diff --git a/tests/queries/0_stateless/01300_group_by_other_keys.sql b/tests/queries/0_stateless/01300_group_by_other_keys.sql new file mode 100644 index 00000000000..d28f3707f7a --- /dev/null +++ b/tests/queries/0_stateless/01300_group_by_other_keys.sql @@ -0,0 +1,30 @@ +set optimize_group_by_function_keys = 1; +set enable_debug_queries = 1; + +SELECT max(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 2, number % 3, (number % 2 + number % 3) % 2 ORDER BY k; +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) ORDER BY k; +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3 ORDER BY k; +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 ORDER BY k; +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) % 3, number % 2 ORDER BY k; + + +analyze SELECT max(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 2, number % 3, (number % 2 + number % 3) % 2 ORDER BY k; +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) ORDER BY k; +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3 ORDER BY k; +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 ORDER BY k; +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) % 3, number % 2 ORDER BY k; + +set optimize_group_by_function_keys = 0; + +SELECT max(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 2, number % 3, (number % 2 + number % 3) % 2 ORDER BY k; +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) ORDER BY k; +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3 ORDER BY k; +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 ORDER BY k; +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) % 3, number % 2 ORDER BY k; + +analyze SELECT max(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 2, number % 3, (number % 2 + number % 3) % 2 ORDER BY k; +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) ORDER BY k; +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3 ORDER BY k; +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 ORDER BY k; +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) % 3, number % 2 ORDER BY k; +-- TODO - test with similar variables of different tables (collision) diff --git a/tests/queries/0_stateless/01300_group_by_other_keys_having.reference b/tests/queries/0_stateless/01300_group_by_other_keys_having.reference new file mode 100644 index 00000000000..d7c04e64df5 --- /dev/null +++ b/tests/queries/0_stateless/01300_group_by_other_keys_having.reference @@ -0,0 +1,24 @@ +3465735.9027997246 +3465735.902799725 +3465736.595946905 +3465734.169931768 +3465734.8630789486 +3465735.5562261306 +0 +1 +4 +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n number % 3,\n number % 2\nHAVING avg(log(2) * number) > 3465735.3\nORDER BY k ASC +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nWHERE ((number % 5) * (number % 5)) < 5\nGROUP BY number % 5\nORDER BY k ASC +SELECT (number % 5) * (number % 5) AS k\nFROM numbers(10000000)\nWHERE ((number % 5) * (number % 5)) < 5\nGROUP BY number % 5\nORDER BY k ASC +3465735.9027997246 +3465735.902799725 +3465736.595946905 +3465734.169931768 +3465734.8630789486 +3465735.5562261306 +0 +1 +4 +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nGROUP BY \n (number % 2) * (number % 3),\n number % 3,\n number % 2\nHAVING avg(log(2) * number) > 3465735.3\nORDER BY k ASC +SELECT avg(log(2) * number) AS k\nFROM numbers(10000000)\nWHERE ((number % 5) * (number % 5)) < 5\nGROUP BY \n number % 5,\n (number % 5) * (number % 5)\nORDER BY k ASC +SELECT (number % 5) * (number % 5) AS k\nFROM numbers(10000000)\nWHERE ((number % 5) * (number % 5)) < 5\nGROUP BY \n number % 5,\n (number % 5) * (number % 5)\nORDER BY k ASC diff --git a/tests/queries/0_stateless/01300_group_by_other_keys_having.sql b/tests/queries/0_stateless/01300_group_by_other_keys_having.sql new file mode 100644 index 00000000000..b359c074c44 --- /dev/null +++ b/tests/queries/0_stateless/01300_group_by_other_keys_having.sql @@ -0,0 +1,21 @@ +set optimize_group_by_function_keys = 1; +set enable_debug_queries = 1; + +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 HAVING avg(log(2) * number) > 3465735.3 ORDER BY k; +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k; +SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k; + + +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 HAVING avg(log(2) * number) > 3465735.3 ORDER BY k; +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k; +analyze SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k; + +set optimize_group_by_function_keys = 0; + +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 HAVING avg(log(2) * number) > 3465735.3 ORDER BY k; +SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k; +SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k; + +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 HAVING avg(log(2) * number) > 3465735.3 ORDER BY k; +analyze SELECT avg(log(2) * number) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k; +analyze SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k; From d5b76b2fe18a7b93f85c640adff932b251c73444 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 16 Jun 2020 13:34:59 +0300 Subject: [PATCH 275/329] Fix parts count in mutations --- src/Storages/MergeTree/ActiveDataPartSet.h | 11 ++++++++ .../MergeTree/ReplicatedMergeTreeQueue.cpp | 25 ++++++++++--------- .../MergeTree/ReplicatedMergeTreeQueue.h | 13 ++++++---- ...076_parallel_alter_replicated_zookeeper.sh | 3 +++ 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/Storages/MergeTree/ActiveDataPartSet.h b/src/Storages/MergeTree/ActiveDataPartSet.h index beb538e2ca5..24d51ce6c84 100644 --- a/src/Storages/MergeTree/ActiveDataPartSet.h +++ b/src/Storages/MergeTree/ActiveDataPartSet.h @@ -67,6 +67,17 @@ public: return result; } + /// Remove only covered parts from active set + bool removePartsCoveredBy(const String & part_name) + { + Strings parts_covered_by = getPartsCoveredBy(MergeTreePartInfo::fromPartName(part_name, format_version)); + bool result = true; + for (const auto & part : parts_covered_by) + result &= remove(part); + + return result; + } + /// If not found, return an empty string. String getContainingPart(const MergeTreePartInfo & part_info) const; String getContainingPart(const String & name) const; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 7cb46ffda7d..56e1b9a5c49 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -208,21 +208,18 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( /// /// But we remove covered parts from mutations, because we actually /// have replacing part. - Strings covered_parts = current_parts.getPartsCoveredBy(MergeTreePartInfo::fromPartName(entry->actual_new_part_name, format_version)); - - for (const auto & covered_part : covered_parts) - removePartFromMutations(covered_part); + /// + /// NOTE actual_new_part_name is very confusing and error-prone. This approach must be fixed. + removeCoveredPartsFromMutations(entry->actual_new_part_name, /*remove_part = */ false); } for (const String & virtual_part_name : entry->getVirtualPartNames()) { - Strings replaced_parts; - current_parts.add(virtual_part_name, &replaced_parts); + current_parts.add(virtual_part_name); /// These parts are already covered by newer part, we don't have to /// mutate it. - for (const auto & replaced_part : replaced_parts) - removePartFromMutations(replaced_part); + removeCoveredPartsFromMutations(virtual_part_name, /*remove_part = */ false); } String drop_range_part_name; @@ -250,13 +247,13 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( /// Because execution of the entry is unsuccessful, /// `virtual_part_name` will never appear so we won't need to mutate /// it. - removePartFromMutations(virtual_part_name); + removeCoveredPartsFromMutations(virtual_part_name, /*remove_part = */ true); } } } -void ReplicatedMergeTreeQueue::removePartFromMutations(const String & part_name) +void ReplicatedMergeTreeQueue::removeCoveredPartsFromMutations(const String & part_name, bool remove_part) { auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); auto in_partition = mutations_by_partition.find(part_info.partition_id); @@ -270,7 +267,11 @@ void ReplicatedMergeTreeQueue::removePartFromMutations(const String & part_name) { MutationStatus & status = *it->second; - status.parts_to_do.removePartAndCoveredParts(part_name); + if (remove_part) + status.parts_to_do.removePartAndCoveredParts(part_name); + else + status.parts_to_do.removePartsCoveredBy(part_name); + if (status.parts_to_do.size() == 0) some_mutations_are_probably_done = true; @@ -1969,6 +1970,6 @@ void ReplicatedMergeTreeQueue::removeCurrentPartsFromMutations() { std::lock_guard state_lock(state_mutex); for (const auto & part_name : current_parts.getParts()) - removePartFromMutations(part_name); + removeCoveredPartsFromMutations(part_name, /*remove_part = */ true); } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 8f3a7719076..7a9dd2d6875 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -209,11 +209,14 @@ private: /// Add part for mutations with block_number > part.getDataVersion() void addPartToMutations(const String & part_name); - /// Remove part from mutations (parts_to_do) which were assigned to mutate it - /// with block_number > part.getDataVersion() - /// and block_number == part.getDataVersion() - /// ^ (this may happen if we downloaded mutated part from other replica) - void removePartFromMutations(const String & part_name); + /// Remove covered parts from mutations (parts_to_do) which were assigned + /// for mutation. If remove_part == true, than also remove part itself. + /// + /// Part removed from mutations which satisfy contitions: + /// block_number > part.getDataVersion() + /// or block_number == part.getDataVersion() + /// ^ (this may happen if we downloaded mutated part from other replica) + void removeCoveredPartsFromMutations(const String & part_name, bool remove_part); /// Update the insertion times in ZooKeeper. void updateTimesInZooKeeper(zkutil::ZooKeeperPtr zookeeper, diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index 4bb8da3d43c..4ae0ed85137 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -110,6 +110,9 @@ while [[ $($CLICKHOUSE_CLIENT --query "select * from system.mutations where tabl break fi sleep 1 + for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_mutate_mt_$i" 2> /dev/null + done counter=$(($counter + 1)) done From be008cd186b903a646cc0931b8be0849e46a6187 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 16 Jun 2020 13:44:23 +0300 Subject: [PATCH 276/329] remove some code --- .../AggregateFunctionSumMap.cpp | 203 ++++++------------ .../AggregateFunctionSumMap.h | 72 +++++-- 2 files changed, 118 insertions(+), 157 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.cpp b/src/AggregateFunctions/AggregateFunctionSumMap.cpp index 0e0d654abf1..9d553ebddb4 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.cpp +++ b/src/AggregateFunctions/AggregateFunctionSumMap.cpp @@ -18,35 +18,6 @@ namespace ErrorCodes namespace { -template -struct SumMap -{ - template - using F = AggregateFunctionSumMap; -}; - -template -struct SumMapFiltered -{ - template - using F = AggregateFunctionSumMapFiltered; -}; - -template -struct MinMap -{ - template - using F = AggregateFunctionMinMap; -}; - -template -struct MaxMap -{ - template - using F = AggregateFunctionMaxMap; -}; - - auto parseArguments(const std::string & name, const DataTypes & arguments) { DataTypes args; @@ -99,97 +70,32 @@ auto parseArguments(const std::string & name, const DataTypes & arguments) tuple_argument}; } -template -AggregateFunctionPtr createAggregateFunctionSumMap(const std::string & name, const DataTypes & arguments, const Array & params) +// This function instantiates a particular overload of the sumMap family of +// functions. +// The template parameter MappedFunction is an aggregate +// function template that allows to choose the aggregate function variant that +// accepts either normal arguments or tuple argument. +template