From 3d3b49d009a9613984e865b9535bfa9418e135e4 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Thu, 22 Oct 2020 16:44:48 +0300 Subject: [PATCH 001/258] Try to parse DataType arguments as another nested one --- src/Parsers/ParserDataType.cpp | 28 +++++++++++++------ .../01532_tuple_with_name_type.reference | 4 +++ .../01532_tuple_with_name_type.sql | 16 +++++++++++ 3 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/01532_tuple_with_name_type.reference create mode 100644 tests/queries/0_stateless/01532_tuple_with_name_type.sql diff --git a/src/Parsers/ParserDataType.cpp b/src/Parsers/ParserDataType.cpp index a0a4eb97efe..9b10111db06 100644 --- a/src/Parsers/ParserDataType.cpp +++ b/src/Parsers/ParserDataType.cpp @@ -1,10 +1,12 @@ #include -#include -#include + #include #include +#include +#include #include + namespace DB { @@ -78,14 +80,24 @@ bool ParserDataType::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ++pos; /// Parse optional parameters - ParserList args_parser(std::make_unique(), std::make_unique(TokenType::Comma)); ASTPtr expr_list_args; - if (!args_parser.parse(pos, expr_list_args, expected)) - return false; - if (pos->type != TokenType::ClosingRoundBracket) - return false; - ++pos; + ParserList args_parser_nested(std::make_unique(), std::make_unique(TokenType::Comma), false); + if (args_parser_nested.parse(pos, expr_list_args, expected)) + { + if (pos->type != TokenType::ClosingRoundBracket) + return false; + ++pos; + } + else + { + ParserList args_parser_expr(std::make_unique(), std::make_unique(TokenType::Comma)); + if (!args_parser_expr.parse(pos, expr_list_args, expected)) + return false; + if (pos->type != TokenType::ClosingRoundBracket) + return false; + ++pos; + } function_node->arguments = expr_list_args; function_node->children.push_back(function_node->arguments); diff --git a/tests/queries/0_stateless/01532_tuple_with_name_type.reference b/tests/queries/0_stateless/01532_tuple_with_name_type.reference new file mode 100644 index 00000000000..065734bf705 --- /dev/null +++ b/tests/queries/0_stateless/01532_tuple_with_name_type.reference @@ -0,0 +1,4 @@ +a Tuple(key String, value String) +a Tuple(Tuple(key String, value String)) +a.key Array(String) +a.value Array(String) diff --git a/tests/queries/0_stateless/01532_tuple_with_name_type.sql b/tests/queries/0_stateless/01532_tuple_with_name_type.sql new file mode 100644 index 00000000000..86e0de330a2 --- /dev/null +++ b/tests/queries/0_stateless/01532_tuple_with_name_type.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS test_01532_1; +DROP TABLE IF EXISTS test_01532_2; +DROP TABLE IF EXISTS test_01532_3; + +CREATE TABLE test_01532_1 (a Tuple(key String, value String)) ENGINE Memory(); +DESCRIBE TABLE test_01532_1; + +CREATE TABLE test_01532_2 (a Tuple(Tuple(key String, value String))) ENGINE Memory(); +DESCRIBE TABLE test_01532_2; + +CREATE TABLE test_01532_3 (a Array(Tuple(key String, value String))) ENGINE Memory(); +DESCRIBE TABLE test_01532_3; + +DROP TABLE test_01532_1; +DROP TABLE test_01532_2; +DROP TABLE test_01532_3; From 15e4e0346972779b99e6c4456f554e52cf5c4edb Mon Sep 17 00:00:00 2001 From: feng lv Date: Sat, 24 Oct 2020 13:18:04 +0000 Subject: [PATCH 002/258] add union distinct and setting union_default_mode --- src/Common/ErrorCodes.cpp | 2 + src/Core/Settings.h | 3 +- src/Core/SettingsEnums.cpp | 6 +++ src/Core/SettingsEnums.h | 10 ++++- .../InterpreterSelectWithUnionQuery.cpp | 37 +++++++++++++++- .../InterpreterSelectWithUnionQuery.h | 4 +- src/Parsers/ASTSelectWithUnionQuery.cpp | 20 +++++++-- src/Parsers/ASTSelectWithUnionQuery.h | 9 ++++ src/Parsers/ParserSelectWithUnionQuery.cpp | 42 +++++++++++++++++-- 9 files changed, 122 insertions(+), 11 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index b841368f662..499b02ca0d9 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -519,6 +519,8 @@ namespace ErrorCodes extern const int CONDITIONAL_TREE_PARENT_NOT_FOUND = 2001; extern const int ILLEGAL_PROJECTION_MANIPULATOR = 2002; extern const int UNRECOGNIZED_ARGUMENTS = 2003; + extern const int UNKNOWN_UNION = 2004; + extern const int EXPECTED_ALL_OR_DISTINCT = 2005; } } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d73098ca6e0..f9431c133e3 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -400,7 +400,8 @@ class IColumn; M(Bool, optimize_trivial_insert_select, true, "Optimize trivial 'INSERT INTO table SELECT ... FROM TABLES' query", 0) \ M(Bool, allow_experimental_database_atomic, true, "Obsolete setting, does nothing. Will be removed after 2021-02-12", 0) \ M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \ - M(Bool, enable_global_with_statement, false, "Propagate WITH statements to UNION queries and all subqueries", 0) + M(Bool, enable_global_with_statement, false, "Propagate WITH statements to UNION queries and all subqueries", 0) \ + M(UnionMode, union_default_mode, UnionMode::ALL, "Set default Union Mode in SelectWithUnion query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without Union Mode will throw exception.", 0) // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS below. diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index c337cd5e3ce..2e1cf025256 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -12,6 +12,7 @@ namespace ErrorCodes extern const int UNKNOWN_JOIN; extern const int BAD_ARGUMENTS; extern const int UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL; + extern const int UNKNOWN_UNION; } @@ -96,4 +97,9 @@ IMPLEMENT_SETTING_MULTI_ENUM(MySQLDataTypesSupport, ErrorCodes::UNKNOWN_MYSQL_DA {{"decimal", MySQLDataTypesSupport::DECIMAL}, {"datetime64", MySQLDataTypesSupport::DATETIME64}}) +IMPLEMENT_SETTING_ENUM(UnionMode, ErrorCodes::UNKNOWN_UNION, + {{"", UnionMode::Unspecified}, + {"ALL", UnionMode::ALL}, + {"DISTINCT", UnionMode::DISTINCT}}) + } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 80b9bf9adde..2546ebe3237 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -119,7 +119,6 @@ enum class DefaultDatabaseEngine DECLARE_SETTING_ENUM(DefaultDatabaseEngine) - enum class MySQLDataTypesSupport { DECIMAL, // convert MySQL's decimal and number to ClickHouse Decimal when applicable @@ -129,4 +128,13 @@ enum class MySQLDataTypesSupport DECLARE_SETTING_MULTI_ENUM(MySQLDataTypesSupport) +enum class UnionMode +{ + Unspecified = 0, // Query UNION without UnionMode will throw exception + ALL, // Query UNION without UnionMode -> SELECT ... UNION ALL SELECT ... + DISTINCT // Query UNION without UnionMode -> SELECT ... UNION DISTINCT SELECT ... +}; + +DECLARE_SETTING_ENUM(UnionMode) + } diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index ba0ebfaaf27..b31723383f7 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB @@ -18,6 +19,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int UNION_ALL_RESULT_STRUCTURES_MISMATCH; + extern const int EXPECTED_ALL_OR_DISTINCT; } @@ -31,13 +33,35 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( context(std::make_shared(context_)), max_streams(context->getSettingsRef().max_threads) { - const auto & ast = query_ptr->as(); + auto & ast = query_ptr->as(); size_t num_selects = ast.list_of_selects->children.size(); if (!num_selects) throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); + /// For SELECT ... UNION/UNION ALL/UNION DISTINCT SELECT ... query, + /// rewrite ast with settings.union_default_mode + if (num_selects > 1) + { + if (ast.mode == ASTSelectWithUnionQuery::Mode::Unspecified) + { + const Settings & settings = context->getSettingsRef(); + if (settings.union_default_mode == UnionMode::ALL) + ast.mode = ASTSelectWithUnionQuery::Mode::ALL; + else if (settings.union_default_mode == UnionMode::DISTINCT) + { + ast.mode = ASTSelectWithUnionQuery::Mode::DISTINCT; + } + else + throw Exception( + "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", + DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); + } + if (ast.mode == ASTSelectWithUnionQuery::Mode::DISTINCT) + distinct_union = true; + } + /// Initialize interpreters for each SELECT query. /// Note that we pass 'required_result_column_names' to first SELECT. /// And for the rest, we pass names at the corresponding positions of 'required_result_column_names' in the result of first SELECT, @@ -197,6 +221,17 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); query_plan.unitePlans(std::move(union_step), std::move(plans)); + + /// Add distinct transform for UNION DISTINCT query + if (distinct_union) + { + const Settings & settings = context->getSettingsRef(); + SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); + + auto distinct_step = std::make_unique(query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false); + + query_plan.addStep(std::move(distinct_step)); + } } BlockIO InterpreterSelectWithUnionQuery::execute() diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h index 5590066a4db..dceba011418 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -12,7 +12,7 @@ class Context; class InterpreterSelectQuery; class QueryPlan; -/** Interprets one or multiple SELECT queries inside UNION ALL chain. +/** Interprets one or multiple SELECT queries inside UNION/UNION ALL/UNION DISTINCT chain. */ class InterpreterSelectWithUnionQuery : public IInterpreter { @@ -54,6 +54,8 @@ private: size_t max_streams = 1; + bool distinct_union = false; + static Block getCommonHeaderForUnion(const Blocks & headers); }; diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 96cac839c58..f2b64b37089 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -23,13 +23,25 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); + std::string mode_str; + switch (mode) + { + case Mode::Unspecified: + mode_str = ""; + break; + case Mode::ALL: + mode_str = "ALL"; + break; + case Mode::DISTINCT: + mode_str = "DISTINCT"; + break; + } + for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) { if (it != list_of_selects->children.begin()) - settings.ostr - << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") - << "UNION ALL" << (settings.hilite ? hilite_none : "") - << settings.nl_or_ws; + settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " << mode_str + << (settings.hilite ? hilite_none : "") << settings.nl_or_ws; (*it)->formatImpl(settings, state, frame); } diff --git a/src/Parsers/ASTSelectWithUnionQuery.h b/src/Parsers/ASTSelectWithUnionQuery.h index 41ec8bb1076..9c476d3674a 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.h +++ b/src/Parsers/ASTSelectWithUnionQuery.h @@ -17,6 +17,15 @@ public: ASTPtr clone() const override; void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + enum class Mode + { + Unspecified, + ALL, + DISTINCT + }; + + Mode mode = Mode::Unspecified; + ASTPtr list_of_selects; }; diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index cebe8ba876d..bf9c9cf552c 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -27,15 +27,51 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & { ASTPtr list_node; - ParserList parser(std::make_unique(), std::make_unique("UNION ALL"), false); - if (!parser.parse(pos, list_node, expected)) - return false; + ParserList parser_union(std::make_unique(), std::make_unique("UNION"), false); + ParserList parser_union_all(std::make_unique(), std::make_unique("UNION ALL"), false); + ParserList parser_union_distinct(std::make_unique(), std::make_unique("UNION DISTINCT"), false); + + auto begin = pos; + auto current_expected = expected; + ASTSelectWithUnionQuery::Mode union_mode = ASTSelectWithUnionQuery::Mode::ALL; + + /// Parser SELECT lists and UNION type, must have UNION + auto union_parser = [&](auto & parser, auto mode) { + if (!parser.parse(pos, list_node, expected)) + { + pos = begin; + expected = current_expected; + return false; + } + /// number of SELECT lists should not less than 2 + if (list_node->children.size() < 2) + { + pos = begin; + expected = current_expected; + return false; + } + union_mode = mode; + return true; + }; + + /// We first parse: SELECT ... UNION SELECT ... + /// SELECT ... UNION ALL SELECT ... + /// SELECT ... UNION DISTINCT SELECT ... + if (!union_parser(parser_union, ASTSelectWithUnionQuery::Mode::Unspecified) + && !union_parser(parser_union_all, ASTSelectWithUnionQuery::Mode::ALL) + && !union_parser(parser_union_distinct, ASTSelectWithUnionQuery::Mode::DISTINCT)) + { + /// If above parse failed, we back to parse SELECT without UNION + if (!parser_union.parse(pos, list_node, expected)) + return false; + } auto select_with_union_query = std::make_shared(); node = select_with_union_query; select_with_union_query->list_of_selects = std::make_shared(); select_with_union_query->children.push_back(select_with_union_query->list_of_selects); + select_with_union_query->mode = union_mode; // flatten inner union query for (auto & child : list_node->children) From 4d0c87816a6eb35e3aa5562f68122477f65d07a0 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sat, 24 Oct 2020 13:22:25 +0000 Subject: [PATCH 003/258] add test --- ...t_and_setting_union_default_mode.reference | 140 ++++++++++++++++++ ...istinct_and_setting_union_default_mode.sql | 13 ++ 2 files changed, 153 insertions(+) create mode 100644 tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference create mode 100644 tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference new file mode 100644 index 00000000000..6c4547333fe --- /dev/null +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference @@ -0,0 +1,140 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql new file mode 100644 index 00000000000..1a6e582aebe --- /dev/null +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql @@ -0,0 +1,13 @@ +SELECT * FROM numbers(10) UNION SELECT * FROM numbers(10); +SELECT * FROM numbers(10) UNION ALL SELECT * FROM numbers(10); +SELECT * FROM numbers(10) UNION DISTINCT SELECT * FROM numbers(10); + +SET union_default_mode='ALL'; +SELECT * FROM numbers(10) UNION SELECT * FROM numbers(10); +SELECT * FROM numbers(10) UNION ALL SELECT * FROM numbers(10); +SELECT * FROM numbers(10) UNION DISTINCT SELECT * FROM numbers(10); + +SET union_default_mode='DISTINCT'; +SELECT * FROM numbers(10) UNION SELECT * FROM numbers(10); +SELECT * FROM numbers(10) UNION ALL SELECT * FROM numbers(10); +SELECT * FROM numbers(10) UNION DISTINCT SELECT * FROM numbers(10); From b2a3ace6d1f2ac802a2acfccad9966eec37284fa Mon Sep 17 00:00:00 2001 From: feng lv Date: Sat, 24 Oct 2020 13:41:20 +0000 Subject: [PATCH 004/258] fix --- src/Core/SettingsEnums.h | 1 + src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 4 ++-- src/Parsers/ASTSelectWithUnionQuery.cpp | 7 +++++-- src/Parsers/ParserSelectWithUnionQuery.cpp | 7 ++++--- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 2546ebe3237..c2ef08135eb 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -119,6 +119,7 @@ enum class DefaultDatabaseEngine DECLARE_SETTING_ENUM(DefaultDatabaseEngine) + enum class MySQLDataTypesSupport { DECIMAL, // convert MySQL's decimal and number to ClickHouse Decimal when applicable diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index b31723383f7..f4ef78dd183 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -47,17 +47,17 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( if (ast.mode == ASTSelectWithUnionQuery::Mode::Unspecified) { const Settings & settings = context->getSettingsRef(); + if (settings.union_default_mode == UnionMode::ALL) ast.mode = ASTSelectWithUnionQuery::Mode::ALL; else if (settings.union_default_mode == UnionMode::DISTINCT) - { ast.mode = ASTSelectWithUnionQuery::Mode::DISTINCT; - } else throw Exception( "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); } + if (ast.mode == ASTSelectWithUnionQuery::Mode::DISTINCT) distinct_union = true; } diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index f2b64b37089..7a61b3b5da1 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -40,8 +40,11 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) { if (it != list_of_selects->children.begin()) - settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " << mode_str - << (settings.hilite ? hilite_none : "") << settings.nl_or_ws; + settings.ostr << settings.nl_or_ws << indent_str + << (settings.hilite ? hilite_keyword : "") + << "UNION " << mode_str + << (settings.hilite ? hilite_none : "") + << settings.nl_or_ws; (*it)->formatImpl(settings, state, frame); } diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index bf9c9cf552c..5b98b083fd4 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -28,15 +28,16 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & ASTPtr list_node; ParserList parser_union(std::make_unique(), std::make_unique("UNION"), false); - ParserList parser_union_all(std::make_unique(), std::make_unique("UNION ALL"), false); - ParserList parser_union_distinct(std::make_unique(), std::make_unique("UNION DISTINCT"), false); + ParserList parser_union_all(std::make_unique(), std::make_unique("UNION ALL"), false); + ParserList parser_union_distinct(std::make_unique(), std::make_unique("UNION DISTINCT"), false); auto begin = pos; auto current_expected = expected; ASTSelectWithUnionQuery::Mode union_mode = ASTSelectWithUnionQuery::Mode::ALL; /// Parser SELECT lists and UNION type, must have UNION - auto union_parser = [&](auto & parser, auto mode) { + auto union_parser = [&](auto & parser, auto mode) + { if (!parser.parse(pos, list_node, expected)) { pos = begin; From dcf4b0df1f20b391963db5ccda93b7a1b1075583 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sat, 24 Oct 2020 14:03:15 +0000 Subject: [PATCH 005/258] fix --- src/Parsers/ASTSelectWithUnionQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 7a61b3b5da1..1ef47257eae 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -40,7 +40,7 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) { if (it != list_of_selects->children.begin()) - settings.ostr << settings.nl_or_ws << indent_str + settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " << mode_str << (settings.hilite ? hilite_none : "") From 5c192df20f974f26f24fcdaf4259468e10c545b7 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sat, 24 Oct 2020 15:40:10 +0000 Subject: [PATCH 006/258] update documents --- docs/en/sql-reference/statements/select/index.md | 4 ++-- .../statements/select/{union-all.md => union.md} | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) rename docs/en/sql-reference/statements/select/{union-all.md => union.md} (64%) diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 07be8c2bf45..42d3e81f226 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -25,7 +25,7 @@ SELECT [DISTINCT] expr_list [ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr] [LIMIT [offset_value, ]n BY columns] [LIMIT [n, ]m] [WITH TIES] -[UNION ALL ...] +[UNION ...] [INTO OUTFILE filename] [FORMAT format] ``` @@ -46,7 +46,7 @@ Specifics of each optional clause are covered in separate sections, which are li - [SELECT clause](#select-clause) - [DISTINCT clause](../../../sql-reference/statements/select/distinct.md) - [LIMIT clause](../../../sql-reference/statements/select/limit.md) -- [UNION ALL clause](../../../sql-reference/statements/select/union-all.md) +- [UNION clause](../../../sql-reference/statements/select/union.md) - [INTO OUTFILE clause](../../../sql-reference/statements/select/into-outfile.md) - [FORMAT clause](../../../sql-reference/statements/select/format.md) diff --git a/docs/en/sql-reference/statements/select/union-all.md b/docs/en/sql-reference/statements/select/union.md similarity index 64% rename from docs/en/sql-reference/statements/select/union-all.md rename to docs/en/sql-reference/statements/select/union.md index 5230363609e..88211858707 100644 --- a/docs/en/sql-reference/statements/select/union-all.md +++ b/docs/en/sql-reference/statements/select/union.md @@ -1,5 +1,5 @@ --- -toc_title: UNION ALL +toc_title: UNION --- # UNION ALL Clause {#union-all-clause} @@ -25,10 +25,13 @@ Type casting is performed for unions. For example, if two queries being combined Queries that are parts of `UNION ALL` can’t be enclosed in round brackets. [ORDER BY](../../../sql-reference/statements/select/order-by.md) and [LIMIT](../../../sql-reference/statements/select/limit.md) are applied to separate queries, not to the final result. If you need to apply a conversion to the final result, you can put all the queries with `UNION ALL` in a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause. -## Limitations {#limitations} +# UNION DISTINCT Clause {#union-distinct-clause} +The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. + +# UNION Clause {#union-clause} +By defaul, `UNION` has same react as `UNION ALL`, but you can specify union mode by setting `union_default_mode`, values can be 'ALL', 'DISTINCT' or empty string. However, if you use `UNION` with setting `union_default_mode` to empty string, it will throw an exception. -Only `UNION ALL` is supported. The regular `UNION` (`UNION DISTINCT`) is not supported. If you need `UNION DISTINCT`, you can write `SELECT DISTINCT` from a subquery containing `UNION ALL`. ## Implementation Details {#implementation-details} -Queries that are parts of `UNION ALL` can be run simultaneously, and their results can be mixed together. +Queries that are parts of `UNION/UNION ALL/UNION DISTINCT` can be run simultaneously, and their results can be mixed together. From a9e4c7144c1d4b139bde4a0c6884eef7bd99e97a Mon Sep 17 00:00:00 2001 From: feng lv Date: Mon, 26 Oct 2020 09:33:34 +0000 Subject: [PATCH 007/258] fix --- .../sql-reference/statements/select/index.md | 2 +- .../select/{union.md => union-all.md} | 2 +- src/Core/Settings.h | 2 +- .../InterpreterSelectWithUnionQuery.cpp | 124 +++++++++++++----- .../InterpreterSelectWithUnionQuery.h | 4 +- src/Parsers/ASTSelectWithUnionQuery.cpp | 26 ++-- src/Parsers/ASTSelectWithUnionQuery.h | 4 +- src/Parsers/ExpressionListParsers.cpp | 46 +++++++ src/Parsers/ExpressionListParsers.h | 48 +++++++ src/Parsers/ParserSelectWithUnionQuery.cpp | 48 ++----- 10 files changed, 215 insertions(+), 91 deletions(-) rename docs/en/sql-reference/statements/select/{union.md => union-all.md} (86%) diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 42d3e81f226..130000cf4b0 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -46,7 +46,7 @@ Specifics of each optional clause are covered in separate sections, which are li - [SELECT clause](#select-clause) - [DISTINCT clause](../../../sql-reference/statements/select/distinct.md) - [LIMIT clause](../../../sql-reference/statements/select/limit.md) -- [UNION clause](../../../sql-reference/statements/select/union.md) +- [UNION clause](../../../sql-reference/statements/select/union-all.md) - [INTO OUTFILE clause](../../../sql-reference/statements/select/into-outfile.md) - [FORMAT clause](../../../sql-reference/statements/select/format.md) diff --git a/docs/en/sql-reference/statements/select/union.md b/docs/en/sql-reference/statements/select/union-all.md similarity index 86% rename from docs/en/sql-reference/statements/select/union.md rename to docs/en/sql-reference/statements/select/union-all.md index 88211858707..1784da37c9a 100644 --- a/docs/en/sql-reference/statements/select/union.md +++ b/docs/en/sql-reference/statements/select/union-all.md @@ -29,7 +29,7 @@ Queries that are parts of `UNION ALL` can’t be enclosed in round brackets. [OR The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. # UNION Clause {#union-clause} -By defaul, `UNION` has same react as `UNION ALL`, but you can specify union mode by setting `union_default_mode`, values can be 'ALL', 'DISTINCT' or empty string. However, if you use `UNION` with setting `union_default_mode` to empty string, it will throw an exception. +By defaul, `UNION` has same react as `UNION DISTINCT`, but you can specify union mode by setting `union_default_mode`, values can be 'ALL', 'DISTINCT' or empty string. However, if you use `UNION` with setting `union_default_mode` to empty string, it will throw an exception. ## Implementation Details {#implementation-details} diff --git a/src/Core/Settings.h b/src/Core/Settings.h index f9431c133e3..50832ecee5c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -401,7 +401,7 @@ class IColumn; M(Bool, allow_experimental_database_atomic, true, "Obsolete setting, does nothing. Will be removed after 2021-02-12", 0) \ M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \ M(Bool, enable_global_with_statement, false, "Propagate WITH statements to UNION queries and all subqueries", 0) \ - M(UnionMode, union_default_mode, UnionMode::ALL, "Set default Union Mode in SelectWithUnion query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without Union Mode will throw exception.", 0) + M(UnionMode, union_default_mode, UnionMode::DISTINCT, "Set default Union Mode in SelectWithUnion query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without Union Mode will throw exception.", 0) // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS below. diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index f4ef78dd183..19f08250430 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include #include @@ -40,26 +42,38 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( if (!num_selects) throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); - /// For SELECT ... UNION/UNION ALL/UNION DISTINCT SELECT ... query, - /// rewrite ast with settings.union_default_mode + /// Rewrite ast with settings.union_default_mode if (num_selects > 1) { - if (ast.mode == ASTSelectWithUnionQuery::Mode::Unspecified) + const Settings & settings = context->getSettingsRef(); + for (auto & mode : ast.union_modes) { - const Settings & settings = context->getSettingsRef(); + if (mode == ASTSelectWithUnionQuery::Mode::Unspecified) + { - if (settings.union_default_mode == UnionMode::ALL) - ast.mode = ASTSelectWithUnionQuery::Mode::ALL; - else if (settings.union_default_mode == UnionMode::DISTINCT) - ast.mode = ASTSelectWithUnionQuery::Mode::DISTINCT; - else - throw Exception( - "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", - DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); + if (settings.union_default_mode == UnionMode::ALL) + mode = ASTSelectWithUnionQuery::Mode::ALL; + else if (settings.union_default_mode == UnionMode::DISTINCT) + mode = ASTSelectWithUnionQuery::Mode::DISTINCT; + else + throw Exception( + "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", + DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); + } + } + /// Optimize: if there is UNION DISTINCT, all previous UNION DISTINCT can be rewritten to UNION ALL. + /// Therefore we have at most one UNION DISTINCT in a sequence. + for (auto rit = ast.union_modes.rbegin(); rit != ast.union_modes.rend(); ++rit) + { + if (*rit == ASTSelectWithUnionQuery::Mode::DISTINCT) + { + /// Number of streams need to do a DISTINCT transform after unite + union_distinct_num = ast.union_modes.rend() - rit + 1; + for (auto mode_to_modify = ++rit; mode_to_modify != ast.union_modes.rend(); ++mode_to_modify) + *mode_to_modify = ASTSelectWithUnionQuery::Mode::ALL; + break; + } } - - if (ast.mode == ASTSelectWithUnionQuery::Mode::DISTINCT) - distinct_union = true; } /// Initialize interpreters for each SELECT query. @@ -207,31 +221,79 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) return; } - std::vector> plans(num_plans); - DataStreams data_streams(num_plans); - - for (size_t i = 0; i < num_plans; ++i) + /// All UNION streams in the chain does not need to do DISTINCT transform + if (union_distinct_num == 0) { - plans[i] = std::make_unique(); - nested_interpreters[i]->buildQueryPlan(*plans[i]); - data_streams[i] = plans[i]->getCurrentDataStream(); + std::vector> plans(num_plans); + DataStreams data_streams(num_plans); + + for (size_t i = 0; i < num_plans; ++i) + { + plans[i] = std::make_unique(); + nested_interpreters[i]->buildQueryPlan(*plans[i]); + data_streams[i] = plans[i]->getCurrentDataStream(); + } + + auto max_threads = context->getSettingsRef().max_threads; + auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); + + query_plan.unitePlans(std::move(union_step), std::move(plans)); } - auto max_threads = context->getSettingsRef().max_threads; - auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); - - query_plan.unitePlans(std::move(union_step), std::move(plans)); - - /// Add distinct transform for UNION DISTINCT query - if (distinct_union) + /// The first union_distinct_num UNION streams need to do a DISTINCT transform after unite + else { + QueryPlan distinct_query_plan; + + std::vector> plans(union_distinct_num); + DataStreams data_streams(union_distinct_num); + + for (size_t i = 0; i < union_distinct_num; ++i) + { + plans[i] = std::make_unique(); + nested_interpreters[i]->buildQueryPlan(*plans[i]); + data_streams[i] = plans[i]->getCurrentDataStream(); + } + + auto max_threads = context->getSettingsRef().max_threads; + auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); + + distinct_query_plan.unitePlans(std::move(union_step), std::move(plans)); + + /// Add distinct transform const Settings & settings = context->getSettingsRef(); SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); - auto distinct_step = std::make_unique(query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false); + auto distinct_step + = std::make_unique(distinct_query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false); - query_plan.addStep(std::move(distinct_step)); + distinct_query_plan.addStep(std::move(distinct_step)); + + /// No other UNION streams after DISTINCT stream + if (num_plans == union_distinct_num) + { + query_plan = std::move(distinct_query_plan); + return; + } + + /// Build final UNION step + std::vector> final_plans(num_plans - union_distinct_num + 1); + DataStreams final_data_streams(num_plans - union_distinct_num + 1); + + final_plans[0] = std::make_unique(std::move(distinct_query_plan)); + final_data_streams[0] = final_plans[0]->getCurrentDataStream(); + + for (size_t i = 1; i < num_plans - union_distinct_num + 1; ++i) + { + final_plans[i] = std::make_unique(); + nested_interpreters[union_distinct_num + i - 1]->buildQueryPlan(*final_plans[i]); + final_data_streams[i] = final_plans[i]->getCurrentDataStream(); + } + + auto final_union_step = std::make_unique(std::move(final_data_streams), result_header, max_threads); + query_plan.unitePlans(std::move(final_union_step), std::move(final_plans)); } + } BlockIO InterpreterSelectWithUnionQuery::execute() diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h index dceba011418..b41f966f336 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -54,7 +55,8 @@ private: size_t max_streams = 1; - bool distinct_union = false; + /// First union_distinct_num streams need to do a DISTINCT transform after unite + size_t union_distinct_num = 0; static Block getCommonHeaderForUnion(const Blocks & headers); }; diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 1ef47257eae..41b34e14571 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -23,27 +23,21 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F { std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); - std::string mode_str; - switch (mode) + auto mode_to_str = [&](auto mode) { - case Mode::Unspecified: - mode_str = ""; - break; - case Mode::ALL: - mode_str = "ALL"; - break; - case Mode::DISTINCT: - mode_str = "DISTINCT"; - break; - } + if (mode == Mode::Unspecified) + return ""; + else if (mode == Mode::ALL) + return "ALL"; + else + return "DISTINCT"; + }; for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) { if (it != list_of_selects->children.begin()) - settings.ostr << settings.nl_or_ws << indent_str - << (settings.hilite ? hilite_keyword : "") - << "UNION " << mode_str - << (settings.hilite ? hilite_none : "") + settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " + << mode_to_str(union_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : "") << settings.nl_or_ws; (*it)->formatImpl(settings, state, frame); diff --git a/src/Parsers/ASTSelectWithUnionQuery.h b/src/Parsers/ASTSelectWithUnionQuery.h index 9c476d3674a..776ff033724 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.h +++ b/src/Parsers/ASTSelectWithUnionQuery.h @@ -24,7 +24,9 @@ public: DISTINCT }; - Mode mode = Mode::Unspecified; + using Modes = std::vector; + + Modes union_modes; ASTPtr list_of_selects; }; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 26affe020b1..b9e5eb2a678 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -99,6 +100,51 @@ bool ParserList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return true; } +bool ParserUnionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ASTs elements; + + auto parse_element = [&] + { + ASTPtr element; + if (!elem_parser->parse(pos, element, expected)) + return false; + + elements.push_back(element); + return true; + }; + + /// Parse UNION type + auto parse_separator = [&] + { + if (s_union_parser->ignore(pos, expected)) + { + // SELECT ... UNION ALL SELECT ... + if (s_all_parser->check(pos, expected)) + { + union_modes.push_back(ASTSelectWithUnionQuery::Mode::ALL); + } + // SELECT ... UNION DISTINCT SELECT ... + else if (s_distinct_parser->check(pos, expected)) + { + union_modes.push_back(ASTSelectWithUnionQuery::Mode::DISTINCT); + } + // SELECT ... UNION SELECT ... + else + union_modes.push_back(ASTSelectWithUnionQuery::Mode::Unspecified); + return true; + } + return false; + }; + + if (!parseUtil(pos, parse_element, parse_separator)) + return false; + + auto list = std::make_shared(result_separator); + list->children = std::move(elements); + node = list; + return true; +} static bool parseOperator(IParser::Pos & pos, const char * op, Expected & expected) { diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index 93a47648a0b..798c127cde7 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -5,6 +5,7 @@ #include #include +#include namespace DB { @@ -73,6 +74,53 @@ private: char result_separator; }; +class ParserUnionList : public IParserBase +{ +public: + ParserUnionList(ParserPtr && elem_parser_, ParserPtr && s_union_parser_, ParserPtr && s_all_parser_, ParserPtr && s_distinct_parser_) + : elem_parser(std::move(elem_parser_)) + , s_union_parser(std::move(s_union_parser_)) + , s_all_parser(std::move(s_all_parser_)) + , s_distinct_parser(std::move(s_distinct_parser_)) + { + } + + template + static bool parseUtil(Pos & pos, const ElemFunc & parse_element, const SepFunc & parse_separator) + { + Pos begin = pos; + if (!parse_element()) + { + pos = begin; + return false; + } + + while (true) + { + begin = pos; + if (!parse_separator() || !parse_element()) + { + pos = begin; + return true; + } + } + + return false; + } + + auto getUnionModes() const { return union_modes; } + +protected: + const char * getName() const override { return "list of union elements"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +private: + ParserPtr elem_parser; + ParserPtr s_union_parser; + ParserPtr s_all_parser; + ParserPtr s_distinct_parser; + char result_separator = ','; + ASTSelectWithUnionQuery::Modes union_modes; +}; /** An expression with an infix binary left-associative operator. * For example, a + b - c + d. diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index 5b98b083fd4..c1e748fbbd6 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -27,52 +28,21 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & { ASTPtr list_node; - ParserList parser_union(std::make_unique(), std::make_unique("UNION"), false); - ParserList parser_union_all(std::make_unique(), std::make_unique("UNION ALL"), false); - ParserList parser_union_distinct(std::make_unique(), std::make_unique("UNION DISTINCT"), false); + ParserUnionList parser( + std::make_unique(), + std::make_unique("UNION"), + std::make_unique("ALL"), + std::make_unique("DISTINCT")); - auto begin = pos; - auto current_expected = expected; - ASTSelectWithUnionQuery::Mode union_mode = ASTSelectWithUnionQuery::Mode::ALL; - - /// Parser SELECT lists and UNION type, must have UNION - auto union_parser = [&](auto & parser, auto mode) - { - if (!parser.parse(pos, list_node, expected)) - { - pos = begin; - expected = current_expected; - return false; - } - /// number of SELECT lists should not less than 2 - if (list_node->children.size() < 2) - { - pos = begin; - expected = current_expected; - return false; - } - union_mode = mode; - return true; - }; - - /// We first parse: SELECT ... UNION SELECT ... - /// SELECT ... UNION ALL SELECT ... - /// SELECT ... UNION DISTINCT SELECT ... - if (!union_parser(parser_union, ASTSelectWithUnionQuery::Mode::Unspecified) - && !union_parser(parser_union_all, ASTSelectWithUnionQuery::Mode::ALL) - && !union_parser(parser_union_distinct, ASTSelectWithUnionQuery::Mode::DISTINCT)) - { - /// If above parse failed, we back to parse SELECT without UNION - if (!parser_union.parse(pos, list_node, expected)) - return false; - } + if (!parser.parse(pos, list_node, expected)) + return false; auto select_with_union_query = std::make_shared(); node = select_with_union_query; select_with_union_query->list_of_selects = std::make_shared(); select_with_union_query->children.push_back(select_with_union_query->list_of_selects); - select_with_union_query->mode = union_mode; + select_with_union_query->union_modes = parser.getUnionModes(); // flatten inner union query for (auto & child : list_node->children) From 7bfd5d9e8deb1f9d4b35f3d6fa309b1f1f311cb6 Mon Sep 17 00:00:00 2001 From: feng lv Date: Wed, 28 Oct 2020 01:29:09 +0000 Subject: [PATCH 008/258] need fix --- .../InterpreterSelectWithUnionQuery.cpp | 367 +++++++++++------- .../InterpreterSelectWithUnionQuery.h | 29 +- src/Interpreters/executeQuery.cpp | 2 + src/Parsers/ASTSelectWithUnionQuery.cpp | 12 +- src/Parsers/ASTSelectWithUnionQuery.h | 8 +- src/Parsers/ExpressionListParsers.cpp | 2 +- src/Parsers/ExpressionListParsers.h | 1 - src/Parsers/IAST.cpp | 5 + src/Parsers/IAST.h | 2 + src/Parsers/ParserSelectWithUnionQuery.cpp | 22 +- ...istinct_and_setting_union_default_mode.sql | 15 +- 11 files changed, 290 insertions(+), 175 deletions(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 19f08250430..92f88342241 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -1,9 +1,6 @@ -#include -#include #include #include #include -#include #include #include #include @@ -13,6 +10,7 @@ #include #include +#include namespace DB { @@ -24,6 +22,153 @@ namespace ErrorCodes extern const int EXPECTED_ALL_OR_DISTINCT; } +struct CustomizeUnionModeRewrite +{ + using TypeToVisit = ASTSelectWithUnionQuery; + + const UnionMode & default_union_mode; + + void visit(ASTSelectWithUnionQuery & union_select, ASTPtr &) + { + size_t num_selects = union_select.list_of_selects->children.size(); + if (!num_selects) + throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); + if (num_selects > 1) + { + for (auto & mode : union_select.union_modes) + { + if (mode == ASTSelectWithUnionQuery::Mode::Unspecified) + { + if (default_union_mode == UnionMode::ALL) + mode = ASTSelectWithUnionQuery::Mode::ALL; + else if (default_union_mode == UnionMode::DISTINCT) + mode = ASTSelectWithUnionQuery::Mode::DISTINCT; + else + throw Exception( + "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", + DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); + } + } + /// Optimize: if there is UNION DISTINCT, all previous UNION DISTINCT can be rewritten to UNION ALL. + /// Therefore we have at most one UNION DISTINCT in a sequence. + for (auto rit = union_select.union_modes.rbegin(); rit != union_select.union_modes.rend(); ++rit) + { + if (*rit == ASTSelectWithUnionQuery::Mode::DISTINCT) + { + /// Number of streams need to do a DISTINCT transform after unite + for (auto mode_to_modify = ++rit; mode_to_modify != union_select.union_modes.rend(); ++mode_to_modify) + *mode_to_modify = ASTSelectWithUnionQuery::Mode::ALL; + break; + } + } + } + } +}; + +using CustomizeUnionQueryOptimizeVisitor = InDepthNodeVisitor, true>; + +QueryPlan NestedInterpreter::buildQueryPlan(const std::shared_ptr & context, const Block & header) +{ + QueryPlan res; + if (type == Type::LEAF) + { + if (interpreter) + { + interpreter->buildQueryPlan(res); + return res; + } + else + throw Exception("Interpreter is not initialized.", ErrorCodes::LOGICAL_ERROR); + } + + if (num_distinct_union == 0) + { + std::vector> plans(children.size()); + DataStreams data_streams(children.size()); + + for (size_t i = 0; i < children.size(); ++i) + { + plans[i] = std::make_unique(children[i]->buildQueryPlan(context, header)); + data_streams[i] = plans[i]->getCurrentDataStream(); + } + + auto max_threads = context->getSettingsRef().max_threads; + auto union_step = std::make_unique(std::move(data_streams), header, max_threads); + + res.unitePlans(std::move(union_step), std::move(plans)); + return res; + } + /// The first union_distinct_num UNION streams need to do a DISTINCT transform after unite + else + { + QueryPlan distinct_query_plan; + + std::vector> plans(num_distinct_union); + DataStreams data_streams(num_distinct_union); + + for (size_t i = 0; i < num_distinct_union; ++i) + { + plans[i] = std::make_unique(children[i]->buildQueryPlan(context, header)); + data_streams[i] = plans[i]->getCurrentDataStream(); + } + + auto max_threads = context->getSettingsRef().max_threads; + auto union_step = std::make_unique(std::move(data_streams), header, max_threads); + + distinct_query_plan.unitePlans(std::move(union_step), std::move(plans)); + + /// Add distinct transform + const Settings & settings = context->getSettingsRef(); + SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); + + auto distinct_step + = std::make_unique(distinct_query_plan.getCurrentDataStream(), limits, 0, header.getNames(), false); + + distinct_query_plan.addStep(std::move(distinct_step)); + + /// No other UNION streams after DISTINCT stream + if (num_distinct_union == children.size()) + { + return distinct_query_plan; + } + + /// Build final UNION step + std::vector> final_plans(children.size() - num_distinct_union + 1); + DataStreams final_data_streams(children.size() - num_distinct_union + 1); + + final_plans[0] = std::make_unique(std::move(distinct_query_plan)); + final_data_streams[0] = final_plans[0]->getCurrentDataStream(); + + for (size_t i = 1; i < children.size() - num_distinct_union + 1; ++i) + { + final_plans[i] = std::make_unique(children[num_distinct_union + i - 1]->buildQueryPlan(context, header)); + final_data_streams[i] = final_plans[i]->getCurrentDataStream(); + } + + auto final_union_step = std::make_unique(std::move(final_data_streams), header, max_threads); + res.unitePlans(std::move(final_union_step), std::move(final_plans)); + return res; + } +} + +void NestedInterpreter::ignoreWithTotals() +{ + if (type == Type::LEAF) + { + if (interpreter) + interpreter->ignoreWithTotals(); + else + { + throw Exception("Interpreter is not initialized.", ErrorCodes::LOGICAL_ERROR); + } + return; + } + for (auto & child : children) + { + child->ignoreWithTotals(); + } +} + InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( const ASTPtr & query_ptr_, @@ -35,71 +180,48 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( context(std::make_shared(context_)), max_streams(context->getSettingsRef().max_threads) { - auto & ast = query_ptr->as(); - - size_t num_selects = ast.list_of_selects->children.size(); - - if (!num_selects) + std::cout << "\n\n In InterpreterSelectWithUnionQuery\n\n"; + const auto & ast = query_ptr->as(); + std::cout << "\n\n before throw\n\n"; + if (!ast.flatten_nodes_list) + std::cout << "\n\n flatten_nodes_list is null\n\n"; + size_t total_num_selects = ast.flatten_nodes_list->children.size(); + std::cout << "\n\n after get num throw\n\n"; + if (!total_num_selects) throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); + std::cout << "\n\n after throw\n\n"; /// Rewrite ast with settings.union_default_mode - if (num_selects > 1) - { - const Settings & settings = context->getSettingsRef(); - for (auto & mode : ast.union_modes) - { - if (mode == ASTSelectWithUnionQuery::Mode::Unspecified) - { + const auto & settings = context->getSettingsRef(); + CustomizeUnionQueryOptimizeVisitor::Data data_union_mode{settings.union_default_mode}; + CustomizeUnionQueryOptimizeVisitor(data_union_mode).visit(query_ptr); - if (settings.union_default_mode == UnionMode::ALL) - mode = ASTSelectWithUnionQuery::Mode::ALL; - else if (settings.union_default_mode == UnionMode::DISTINCT) - mode = ASTSelectWithUnionQuery::Mode::DISTINCT; - else - throw Exception( - "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", - DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); - } - } - /// Optimize: if there is UNION DISTINCT, all previous UNION DISTINCT can be rewritten to UNION ALL. - /// Therefore we have at most one UNION DISTINCT in a sequence. - for (auto rit = ast.union_modes.rbegin(); rit != ast.union_modes.rend(); ++rit) - { - if (*rit == ASTSelectWithUnionQuery::Mode::DISTINCT) - { - /// Number of streams need to do a DISTINCT transform after unite - union_distinct_num = ast.union_modes.rend() - rit + 1; - for (auto mode_to_modify = ++rit; mode_to_modify != ast.union_modes.rend(); ++mode_to_modify) - *mode_to_modify = ASTSelectWithUnionQuery::Mode::ALL; - break; - } - } - } - - /// Initialize interpreters for each SELECT query. + /// We first build nested interpreters for each select query, then using this nested interpreters to build Tree Structured nested interpreter. /// Note that we pass 'required_result_column_names' to first SELECT. /// And for the rest, we pass names at the corresponding positions of 'required_result_column_names' in the result of first SELECT, /// because names could be different. - nested_interpreters.reserve(num_selects); - - std::vector required_result_column_names_for_other_selects(num_selects); - if (!required_result_column_names.empty() && num_selects > 1) + std::vector> interpreters; + interpreters.reserve(total_num_selects); + std::vector required_result_column_names_for_other_selects(total_num_selects); + if (!required_result_column_names.empty() && total_num_selects > 1) { /// Result header if there are no filtering by 'required_result_column_names'. /// We use it to determine positions of 'required_result_column_names' in SELECT clause. - Block full_result_header = InterpreterSelectQuery( - ast.list_of_selects->children.at(0), *context, options.copy().analyze().noModify()).getSampleBlock(); + Block full_result_header + = InterpreterSelectQuery(ast.flatten_nodes_list->children.at(0), *context, options.copy().analyze().noModify()) + .getSampleBlock(); std::vector positions_of_required_result_columns(required_result_column_names.size()); for (size_t required_result_num = 0, size = required_result_column_names.size(); required_result_num < size; ++required_result_num) positions_of_required_result_columns[required_result_num] = full_result_header.getPositionByName(required_result_column_names[required_result_num]); - for (size_t query_num = 1; query_num < num_selects; ++query_num) + for (size_t query_num = 1; query_num < total_num_selects; ++query_num) { - Block full_result_header_for_current_select = InterpreterSelectQuery( - ast.list_of_selects->children.at(query_num), *context, options.copy().analyze().noModify()).getSampleBlock(); + Block full_result_header_for_current_select + = InterpreterSelectQuery(ast.flatten_nodes_list->children.at(query_num), *context, options.copy().analyze().noModify()) + .getSampleBlock(); if (full_result_header_for_current_select.columns() != full_result_header.columns()) throw Exception("Different number of columns in UNION ALL elements:\n" @@ -114,29 +236,26 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( } } - for (size_t query_num = 0; query_num < num_selects; ++query_num) + for (size_t query_num = 0; query_num < total_num_selects; ++query_num) { const Names & current_required_result_column_names = query_num == 0 ? required_result_column_names : required_result_column_names_for_other_selects[query_num]; - nested_interpreters.emplace_back(std::make_unique( - ast.list_of_selects->children.at(query_num), - *context, - options, - current_required_result_column_names)); + interpreters.emplace_back(std::make_shared( + ast.flatten_nodes_list->children.at(query_num), *context, options, current_required_result_column_names)); } /// Determine structure of the result. - if (num_selects == 1) + if (total_num_selects == 1) { - result_header = nested_interpreters.front()->getSampleBlock(); + result_header = interpreters.front()->getSampleBlock(); } else { - Blocks headers(num_selects); - for (size_t query_num = 0; query_num < num_selects; ++query_num) - headers[query_num] = nested_interpreters[query_num]->getSampleBlock(); + Blocks headers(total_num_selects); + for (size_t query_num = 0; query_num < total_num_selects; ++query_num) + headers[query_num] = interpreters[query_num]->getSampleBlock(); result_header = getCommonHeaderForUnion(headers); } @@ -144,7 +263,7 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( /// InterpreterSelectWithUnionQuery ignores limits if all nested interpreters ignore limits. bool all_nested_ignore_limits = true; bool all_nested_ignore_quota = true; - for (auto & interpreter : nested_interpreters) + for (auto & interpreter : interpreters) { if (!interpreter->ignoreLimits()) all_nested_ignore_limits = false; @@ -153,8 +272,46 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( } options.ignore_limits |= all_nested_ignore_limits; options.ignore_quota |= all_nested_ignore_quota; + + int index = 0; + buildNestedTreeInterpreter(query_ptr, nested_interpreter, interpreters, index); } +/// We build a Tree Structured nested interpreters to build QueryPlan later +/// The structure of build nested interpreters is same as AST Tree +void InterpreterSelectWithUnionQuery::buildNestedTreeInterpreter( + const ASTPtr & ast_ptr, + std::shared_ptr nested_interpreter_, + std::vector> & interpreters, + int & index) +{ + std::cout << "\n\n in build \n\n"; + if (auto inner_union = ast_ptr->as()) + { + auto internal_intepreter = std::make_shared(); + const auto & union_modes = inner_union->union_modes; + + for (auto rit = union_modes.rbegin(); rit != union_modes.rend(); ++rit) + { + if (*rit == ASTSelectWithUnionQuery::Mode::DISTINCT) + { + internal_intepreter->num_distinct_union = union_modes.rend() - rit + 1; + break; + } + } + + nested_interpreter_->children.push_back(internal_intepreter); + + for (auto & child : inner_union->list_of_selects->children) + buildNestedTreeInterpreter(child, internal_intepreter, interpreters, index); + return; + } + + auto leaf_interpreter = std::make_shared(); + leaf_interpreter->type = NestedInterpreter::Type::LEAF; + leaf_interpreter->interpreter = interpreters[index++]; + nested_interpreter_->children.push_back(leaf_interpreter); +} Block InterpreterSelectWithUnionQuery::getCommonHeaderForUnion(const Blocks & headers) { @@ -212,88 +369,7 @@ Block InterpreterSelectWithUnionQuery::getSampleBlock( void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) { - size_t num_plans = nested_interpreters.size(); - - /// Skip union for single interpreter. - if (num_plans == 1) - { - nested_interpreters.front()->buildQueryPlan(query_plan); - return; - } - - /// All UNION streams in the chain does not need to do DISTINCT transform - if (union_distinct_num == 0) - { - std::vector> plans(num_plans); - DataStreams data_streams(num_plans); - - for (size_t i = 0; i < num_plans; ++i) - { - plans[i] = std::make_unique(); - nested_interpreters[i]->buildQueryPlan(*plans[i]); - data_streams[i] = plans[i]->getCurrentDataStream(); - } - - auto max_threads = context->getSettingsRef().max_threads; - auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); - - query_plan.unitePlans(std::move(union_step), std::move(plans)); - } - - /// The first union_distinct_num UNION streams need to do a DISTINCT transform after unite - else - { - QueryPlan distinct_query_plan; - - std::vector> plans(union_distinct_num); - DataStreams data_streams(union_distinct_num); - - for (size_t i = 0; i < union_distinct_num; ++i) - { - plans[i] = std::make_unique(); - nested_interpreters[i]->buildQueryPlan(*plans[i]); - data_streams[i] = plans[i]->getCurrentDataStream(); - } - - auto max_threads = context->getSettingsRef().max_threads; - auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); - - distinct_query_plan.unitePlans(std::move(union_step), std::move(plans)); - - /// Add distinct transform - const Settings & settings = context->getSettingsRef(); - SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); - - auto distinct_step - = std::make_unique(distinct_query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false); - - distinct_query_plan.addStep(std::move(distinct_step)); - - /// No other UNION streams after DISTINCT stream - if (num_plans == union_distinct_num) - { - query_plan = std::move(distinct_query_plan); - return; - } - - /// Build final UNION step - std::vector> final_plans(num_plans - union_distinct_num + 1); - DataStreams final_data_streams(num_plans - union_distinct_num + 1); - - final_plans[0] = std::make_unique(std::move(distinct_query_plan)); - final_data_streams[0] = final_plans[0]->getCurrentDataStream(); - - for (size_t i = 1; i < num_plans - union_distinct_num + 1; ++i) - { - final_plans[i] = std::make_unique(); - nested_interpreters[union_distinct_num + i - 1]->buildQueryPlan(*final_plans[i]); - final_data_streams[i] = final_plans[i]->getCurrentDataStream(); - } - - auto final_union_step = std::make_unique(std::move(final_data_streams), result_header, max_threads); - query_plan.unitePlans(std::move(final_union_step), std::move(final_plans)); - } - + query_plan = nested_interpreter->buildQueryPlan(context, result_header); } BlockIO InterpreterSelectWithUnionQuery::execute() @@ -314,8 +390,7 @@ BlockIO InterpreterSelectWithUnionQuery::execute() void InterpreterSelectWithUnionQuery::ignoreWithTotals() { - for (auto & interpreter : nested_interpreters) - interpreter->ignoreWithTotals(); + nested_interpreter->ignoreWithTotals(); } } diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h index b41f966f336..4af73b3c723 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -1,10 +1,10 @@ #pragma once -#include #include #include #include #include +#include namespace DB { @@ -13,6 +13,22 @@ class Context; class InterpreterSelectQuery; class QueryPlan; +struct NestedInterpreter +{ + ~NestedInterpreter() { } + enum class Type + { + LEAF, + INTERNAL + }; + Type type = Type::INTERNAL; + std::vector> children; + std::shared_ptr interpreter; + size_t num_distinct_union = 0; + QueryPlan buildQueryPlan(const std::shared_ptr & context, const Block & header); + void ignoreWithTotals(); +}; + /** Interprets one or multiple SELECT queries inside UNION/UNION ALL/UNION DISTINCT chain. */ class InterpreterSelectWithUnionQuery : public IInterpreter @@ -49,16 +65,19 @@ private: ASTPtr query_ptr; std::shared_ptr context; - std::vector> nested_interpreters; + std::shared_ptr nested_interpreter; Block result_header; size_t max_streams = 1; - /// First union_distinct_num streams need to do a DISTINCT transform after unite - size_t union_distinct_num = 0; - static Block getCommonHeaderForUnion(const Blocks & headers); + + static void buildNestedTreeInterpreter( + const ASTPtr & ast_ptr, + std::shared_ptr nested_interpreter_, + std::vector> & interpreters, + int & index); }; } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 57c557c5658..8b4b35785c1 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -70,10 +70,12 @@ namespace ErrorCodes static void checkASTSizeLimits(const IAST & ast, const Settings & settings) { + std::cout << "\n\n before check limits"; if (settings.max_ast_depth) ast.checkDepth(settings.max_ast_depth); if (settings.max_ast_elements) ast.checkSize(settings.max_ast_elements); + std::cout << "\n\n after check limits"; } diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 41b34e14571..5deae6f653f 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace DB { @@ -14,6 +15,9 @@ ASTPtr ASTSelectWithUnionQuery::clone() const res->list_of_selects = list_of_selects->clone(); res->children.push_back(res->list_of_selects); + res->union_modes.insert(res->union_modes.begin(), union_modes.begin(), union_modes.end()); + res->flatten_nodes_list = flatten_nodes_list->clone(); + cloneOutputOptions(*res); return res; } @@ -21,8 +25,10 @@ ASTPtr ASTSelectWithUnionQuery::clone() const void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { + std::cout << "\n\nin format \n\n"; std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); +#if 0 auto mode_to_str = [&](auto mode) { if (mode == Mode::Unspecified) @@ -32,16 +38,18 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F else return "DISTINCT"; }; +#endif - for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) + for (ASTs::const_iterator it = flatten_nodes_list->children.begin(); it != flatten_nodes_list->children.end(); ++it) { if (it != list_of_selects->children.begin()) settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " - << mode_to_str(union_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : "") + // << mode_to_str(union_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : "") << settings.nl_or_ws; (*it)->formatImpl(settings, state, frame); } + std::cout << "\n\nafter format \n\n"; } } diff --git a/src/Parsers/ASTSelectWithUnionQuery.h b/src/Parsers/ASTSelectWithUnionQuery.h index 776ff033724..67ec21e246c 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.h +++ b/src/Parsers/ASTSelectWithUnionQuery.h @@ -5,9 +5,8 @@ namespace DB { - -/** Single SELECT query or multiple SELECT queries with UNION ALL. - * Only UNION ALL is possible. No UNION DISTINCT or plain UNION. +/** Single SELECT query or multiple SELECT queries with UNION + * or UNION or UNION DISTINCT */ class ASTSelectWithUnionQuery : public ASTQueryWithOutput { @@ -29,6 +28,9 @@ public: Modes union_modes; ASTPtr list_of_selects; + + /// we need flatten_nodes to help build nested_interpreter + ASTPtr flatten_nodes_list; }; } diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 8b9a36422af..220d304751e 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -139,7 +139,7 @@ bool ParserUnionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!parseUtil(pos, parse_element, parse_separator)) return false; - auto list = std::make_shared(result_separator); + auto list = std::make_shared(); list->children = std::move(elements); node = list; return true; diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index a6f1be712c4..d93952923a9 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -119,7 +119,6 @@ private: ParserPtr s_union_parser; ParserPtr s_all_parser; ParserPtr s_distinct_parser; - char result_separator = ','; ASTSelectWithUnionQuery::Modes union_modes; }; diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index 8ee4154541b..d9f0b3562bc 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -76,13 +76,18 @@ void IAST::updateTreeHashImpl(SipHash & hash_state) const size_t IAST::checkDepthImpl(size_t max_depth, size_t level) const { + std::cout << "\n\n in check depth impl\n\n"; + std::cout << "\nchildren.size = " << children.size() << "\n\n"; size_t res = level + 1; for (const auto & child : children) { + std::cout << "\n in for\n\n"; if (level >= max_depth) throw Exception("AST is too deep. Maximum: " + toString(max_depth), ErrorCodes::TOO_DEEP_AST); res = std::max(res, child->checkDepthImpl(max_depth, level + 1)); + std::cout << "\n after for\n\n"; } + std::cout << "\n\n after impl\n\n"; return res; } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index c88c80021d6..d1fca853592 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -11,6 +11,7 @@ #include #include +#include class SipHash; @@ -91,6 +92,7 @@ public: */ size_t checkDepth(size_t max_depth) const { + std::cout << "\n in check depth\n\n"; return checkDepthImpl(max_depth, 0); } diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index c1e748fbbd6..ee03da753e4 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -9,13 +10,14 @@ namespace DB { - static void getSelectsFromUnionListNode(ASTPtr & ast_select, ASTs & selects) { if (auto * inner_union = ast_select->as()) { for (auto & child : inner_union->list_of_selects->children) + { getSelectsFromUnionListNode(child, selects); + } return; } @@ -23,9 +25,9 @@ static void getSelectsFromUnionListNode(ASTPtr & ast_select, ASTs & selects) selects.push_back(std::move(ast_select)); } - bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { + std::cout << "\n\n in ParserSelectWithUnionQuery\n\n"; ASTPtr list_node; ParserUnionList parser( @@ -42,11 +44,23 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & node = select_with_union_query; select_with_union_query->list_of_selects = std::make_shared(); select_with_union_query->children.push_back(select_with_union_query->list_of_selects); + select_with_union_query->list_of_selects->children.insert( + select_with_union_query->list_of_selects->children.begin(), list_node->children.begin(), list_node->children.end()); select_with_union_query->union_modes = parser.getUnionModes(); - // flatten inner union query + /// NOTE: We cann't simply flatten inner union query now, since we may have different union mode in query, + /// so flatten may change it's semantics. For example: + /// flatten `SELECT 1 UNION (SELECT 1 UNION ALL SELETC 1)` -> `SELECT 1 UNION SELECT 1 UNION ALL SELECT 1` + /// We can use a non-flatten AST to help build QueryPlan in InterpreterSelectWithUnionQuery + + select_with_union_query->flatten_nodes_list = std::make_shared(); + for (auto & child : list_node->children) - getSelectsFromUnionListNode(child, select_with_union_query->list_of_selects->children); + { + getSelectsFromUnionListNode(child, select_with_union_query->flatten_nodes_list->children); + } + std::cout << "\n\n after ParserSelectWithUnionQuery\n\n"; + std::cout << "\n\n flatten_nodes.size =" << select_with_union_query->flatten_nodes_list->children.size() << "\n\n"; return true; } diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql index 1a6e582aebe..12fe204591c 100644 --- a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql @@ -1,13 +1,2 @@ -SELECT * FROM numbers(10) UNION SELECT * FROM numbers(10); -SELECT * FROM numbers(10) UNION ALL SELECT * FROM numbers(10); -SELECT * FROM numbers(10) UNION DISTINCT SELECT * FROM numbers(10); - -SET union_default_mode='ALL'; -SELECT * FROM numbers(10) UNION SELECT * FROM numbers(10); -SELECT * FROM numbers(10) UNION ALL SELECT * FROM numbers(10); -SELECT * FROM numbers(10) UNION DISTINCT SELECT * FROM numbers(10); - -SET union_default_mode='DISTINCT'; -SELECT * FROM numbers(10) UNION SELECT * FROM numbers(10); -SELECT * FROM numbers(10) UNION ALL SELECT * FROM numbers(10); -SELECT * FROM numbers(10) UNION DISTINCT SELECT * FROM numbers(10); +SELECT 1; +(((((((SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1; From 18fbb3dc23212ed03d6788aff612c3a7eac405f6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 30 Oct 2020 00:29:10 +0300 Subject: [PATCH 009/258] some drafts --- .../AggregateFunctionSumMap.h | 2 +- src/Core/Settings.h | 2 + src/DataTypes/DataTypeTuple.cpp | 126 ++++++++++--- src/Formats/FormatFactory.cpp | 2 + src/Formats/FormatSettings.h | 4 + src/Functions/tuple.cpp | 2 +- src/IO/HTTPCommon.cpp | 5 +- src/IO/ReadHelpers.cpp | 8 +- src/Parsers/ExpressionElementParsers.cpp | 1 + .../Impl/JSONEachRowRowOutputFormat.cpp | 67 ++++++- .../Formats/Impl/JSONEachRowRowOutputFormat.h | 9 +- ...JSONEachRowWithProgressRowOutputFormat.cpp | 14 +- .../Formats/Impl/JSONRowOutputFormat.cpp | 166 +++++++++++++----- .../Formats/Impl/JSONRowOutputFormat.h | 13 +- src/Storages/StorageFile.cpp | 13 ++ 15 files changed, 343 insertions(+), 91 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index 456334ee9c3..6533903bbc1 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -105,7 +105,7 @@ public: types.emplace_back(std::make_shared(result_type)); } - return std::make_shared(types); + return std::make_shared(types, Strings{"keys", "values"}); } static const auto & getArgumentColumns(const IColumn**& columns) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index fdc1ba4b28a..791ed49b7c7 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -439,6 +439,8 @@ class IColumn; M(Bool, output_format_json_quote_denormals, false, "Enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format.", 0) \ \ M(Bool, output_format_json_escape_forward_slashes, true, "Controls escaping forward slashes for string outputs in JSON output format. This is intended for compatibility with JavaScript. Don't confuse with backslashes that are always escaped.", 0) \ + M(Bool, output_format_json_write_metadata, true, "Write metadata in JSON output format.", 0) \ + M(Bool, output_format_json_named_tuple_as_object, false, "Serialize named tuple columns as JSON objects.", 0) \ \ M(UInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.", 0) \ M(UInt64, output_format_pretty_max_column_pad_width, 250, "Maximum width to pad all values in a column in Pretty formats.", 0) \ diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp index 453cb7f37a3..f37e28dcef4 100644 --- a/src/DataTypes/DataTypeTuple.cpp +++ b/src/DataTypes/DataTypeTuple.cpp @@ -25,12 +25,13 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; - extern const int EMPTY_DATA_PASSED; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int DUPLICATE_COLUMN; extern const int BAD_ARGUMENTS; + extern const int DUPLICATE_COLUMN; + extern const int EMPTY_DATA_PASSED; + extern const int LOGICAL_ERROR; extern const int NOT_FOUND_COLUMN_IN_BLOCK; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH; } @@ -145,6 +146,20 @@ static void addElementSafe(const DataTypes & elems, IColumn & column, F && impl) try { impl(); + + // Check that all columns now have the same size. + size_t new_size = column.size(); + for (auto i : ext::range(1, ext::size(elems))) + { + const auto & element_column = extractElementColumn(column, i); + if (element_column.size() != new_size) + { + // This is not a logical error because it may work with + // user-supplied data. + throw Exception(ErrorCodes::SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH, + "Cannot read a tuple because not all elements are present"); + } + } } catch (...) { @@ -157,6 +172,7 @@ static void addElementSafe(const DataTypes & elems, IColumn & column, F && impl) throw; } + } @@ -213,37 +229,93 @@ void DataTypeTuple::deserializeText(IColumn & column, ReadBuffer & istr, const F void DataTypeTuple::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { - writeChar('[', ostr); - for (const auto i : ext::range(0, ext::size(elems))) + if (settings.json.named_tuple_as_object + && have_explicit_names) { - if (i != 0) - writeChar(',', ostr); - elems[i]->serializeAsTextJSON(extractElementColumn(column, i), row_num, ostr, settings); + writeChar('{', ostr); + for (const auto i : ext::range(0, ext::size(elems))) + { + if (i != 0) + { + writeChar(',', ostr); + } + writeJSONString(names[i], ostr, settings); + writeChar(':', ostr); + elems[i]->serializeAsTextJSON(extractElementColumn(column, i), row_num, ostr, settings); + } + writeChar('}', ostr); + } + else + { + writeChar('[', ostr); + for (const auto i : ext::range(0, ext::size(elems))) + { + if (i != 0) + writeChar(',', ostr); + elems[i]->serializeAsTextJSON(extractElementColumn(column, i), row_num, ostr, settings); + } + writeChar(']', ostr); } - writeChar(']', ostr); } void DataTypeTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - const size_t size = elems.size(); - assertChar('[', istr); - - addElementSafe(elems, column, [&] + if (settings.json.named_tuple_as_object + && have_explicit_names) { - for (const auto i : ext::range(0, size)) - { - skipWhitespaceIfAny(istr); - if (i != 0) - { - assertChar(',', istr); - skipWhitespaceIfAny(istr); - } - elems[i]->deserializeAsTextJSON(extractElementColumn(column, i), istr, settings); - } - }); + skipWhitespaceIfAny(istr); + assertChar('{', istr); + skipWhitespaceIfAny(istr); - skipWhitespaceIfAny(istr); - assertChar(']', istr); + addElementSafe(elems, column, [&] + { + // Require all elements but in arbitrary order. + for (auto i : ext::range(0, ext::size(elems))) + { + if (i > 0) + { + skipWhitespaceIfAny(istr); + assertChar(',', istr); + skipWhitespaceIfAny(istr); + } + + std::string name; + readDoubleQuotedString(name, istr); + skipWhitespaceIfAny(istr); + assertChar(':', istr); + skipWhitespaceIfAny(istr); + + const size_t element_pos = getPositionByName(name); + auto & element_column = extractElementColumn(column, element_pos); + elems[element_pos]->deserializeAsTextJSON(element_column, istr, settings); + } + }); + + skipWhitespaceIfAny(istr); + assertChar('}', istr); + } + else + { + const size_t size = elems.size(); + assertChar('[', istr); + + addElementSafe(elems, column, [&] + { + for (const auto i : ext::range(0, size)) + { + skipWhitespaceIfAny(istr); + if (i != 0) + { + assertChar(',', istr); + skipWhitespaceIfAny(istr); + } + elems[i]->deserializeAsTextJSON(extractElementColumn(column, i), istr, settings); + } + }); + + skipWhitespaceIfAny(istr); + assertChar(']', istr); + } } void DataTypeTuple::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 4dc5b816420..d7601166136 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -99,6 +99,8 @@ static FormatSettings getOutputFormatSetting(const Settings & settings, const Co format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes; + format_settings.json.write_metadata = settings.output_format_json_write_metadata; + //format_settings.json.named_tuple_as_object = settings.output_format_json_named_tuple_as_object; format_settings.csv.delimiter = settings.format_csv_delimiter; format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 8d7c3cdb49f..102ae4a5a24 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -22,6 +22,10 @@ struct FormatSettings bool quote_64bit_integers = true; bool quote_denormals = true; bool escape_forward_slashes = true; + bool write_metadata = false; + bool named_tuple_as_object = true; + bool list_of_rows = false; + bool serialize_as_strings = false; }; JSON json; diff --git a/src/Functions/tuple.cpp b/src/Functions/tuple.cpp index 6d24391ed46..6522723dde4 100644 --- a/src/Functions/tuple.cpp +++ b/src/Functions/tuple.cpp @@ -68,7 +68,7 @@ public: /// Create named tuple if possible. if (DataTypeTuple::canBeCreatedWithNames(names)) - return std::make_shared(types, names, false); + return std::make_shared(types, names); return std::make_shared(types); } diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 6b7f30cd9b6..7c3242ba0c8 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -236,7 +236,10 @@ void assertResponseIsOk(const Poco::Net::HTTPRequest & request, Poco::Net::HTTPR { auto status = response.getStatus(); - if (!(status == Poco::Net::HTTPResponse::HTTP_OK || (isRedirect(status) && allow_redirects))) + if (!(status == Poco::Net::HTTPResponse::HTTP_OK + || status == Poco::Net::HTTPResponse::HTTP_CREATED + || status == Poco::Net::HTTPResponse::HTTP_ACCEPTED + || (isRedirect(status) && allow_redirects))) { std::stringstream error_message; error_message << "Received error from remote server " << request.getURI() << ". HTTP status code: " << status << " " diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 900e9c7b535..3ff8158096e 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -493,8 +493,12 @@ template static void readAnyQuotedStringInto(Vector & s, ReadBuffer & buf) { if (buf.eof() || *buf.position() != quote) - throw Exception("Cannot parse quoted string: expected opening quote", - ErrorCodes::CANNOT_PARSE_QUOTED_STRING); + { + throw Exception(ErrorCodes::CANNOT_PARSE_QUOTED_STRING, + "Cannot parse quoted string: expected opening quote '{}', got '{}'", + std::string{quote}, buf.eof() ? "EOF" : std::string{*buf.position()}); + } + ++buf.position(); while (!buf.eof()) diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 3c45bd005a9..8d25609a7a7 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1475,6 +1475,7 @@ bool ParserExpressionElement::parseImpl(Pos & pos, ASTPtr & node, Expected & exp || ParserFunction().parse(pos, node, expected) || ParserQualifiedAsterisk().parse(pos, node, expected) || ParserAsterisk().parse(pos, node, expected) + //|| ParserTupleElementExpression().parse(pos, node, expected) || ParserCompoundIdentifier().parse(pos, node, expected) || ParserSubstitution().parse(pos, node, expected) || ParserMySQLGlobalVariable().parse(pos, node, expected); diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp index b3b83949642..2459c6f2451 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp @@ -12,9 +12,9 @@ JSONEachRowRowOutputFormat::JSONEachRowRowOutputFormat( WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_, - const FormatSettings & settings_, - bool yield_strings_) - : IRowOutputFormat(header_, out_, params_), settings(settings_), yield_strings(yield_strings_) + const FormatSettings & settings_) + : IRowOutputFormat(header_, out_, params_), + settings(settings_) { const auto & sample = getPort(PortKind::Main).getHeader(); size_t columns = sample.columns(); @@ -33,7 +33,7 @@ void JSONEachRowRowOutputFormat::writeField(const IColumn & column, const IDataT writeString(fields[field_number], out); writeChar(':', out); - if (yield_strings) + if (settings.json.serialize_as_strings) { WriteBufferFromOwnString buf; @@ -61,29 +61,78 @@ void JSONEachRowRowOutputFormat::writeRowStartDelimiter() void JSONEachRowRowOutputFormat::writeRowEndDelimiter() { - writeCString("}\n", out); + writeCString("}", out); field_number = 0; } +void JSONEachRowRowOutputFormat::writeRowBetweenDelimiter() +{ + if (settings.json.list_of_rows) + { + writeCString(",\n", out); + } + else + { + writeCString("\n", out); + } +} + + +void JSONEachRowRowOutputFormat::writePrefix() +{ + if (settings.json.list_of_rows) + { + writeCString("[\n", out); + } +} + + +void JSONEachRowRowOutputFormat::writeSuffix() +{ + if (settings.json.list_of_rows) + { + writeCString("\n]\n", out); + } +} + + void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory) { factory.registerOutputFormatProcessor("JSONEachRow", []( WriteBuffer & buf, const Block & sample, const RowOutputFormatParams & params, - const FormatSettings & format_settings) + const FormatSettings & _format_settings) { - return std::make_shared(buf, sample, params, format_settings, false); + FormatSettings settings = _format_settings; + settings.json.serialize_as_strings = false; + return std::make_shared(buf, sample, params, + settings); }); factory.registerOutputFormatProcessor("JSONStringsEachRow", []( WriteBuffer & buf, const Block & sample, const RowOutputFormatParams & params, - const FormatSettings & format_settings) + const FormatSettings & _format_settings) { - return std::make_shared(buf, sample, params, format_settings, true); + FormatSettings settings = _format_settings; + settings.json.serialize_as_strings = true; + return std::make_shared(buf, sample, params, + settings); + }); + + factory.registerOutputFormatProcessor("JSONList", []( + WriteBuffer & buf, + const Block & sample, + const RowOutputFormatParams & params, + const FormatSettings & _format_settings) + { + FormatSettings settings = _format_settings; + settings.json.list_of_rows = true; + return std::make_shared(buf, sample, params, + settings); }); } diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h index bd9cfff68c5..38760379056 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h @@ -19,8 +19,7 @@ public: WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_, - const FormatSettings & settings_, - bool yield_strings_); + const FormatSettings & settings_); String getName() const override { return "JSONEachRowRowOutputFormat"; } @@ -28,6 +27,9 @@ public: void writeFieldDelimiter() override; void writeRowStartDelimiter() override; void writeRowEndDelimiter() override; + void writeRowBetweenDelimiter() override; + void writePrefix() override; + void writeSuffix() override; protected: /// No totals and extremes. @@ -40,9 +42,6 @@ private: Names fields; FormatSettings settings; - -protected: - bool yield_strings; }; } diff --git a/src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp index 48c23abd680..4612ce99f05 100644 --- a/src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp @@ -34,18 +34,24 @@ void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factor WriteBuffer & buf, const Block & sample, const RowOutputFormatParams & params, - const FormatSettings & format_settings) + const FormatSettings & _format_settings) { - return std::make_shared(buf, sample, params, format_settings, false); + FormatSettings settings = _format_settings; + settings.json.serialize_as_strings = false; + return std::make_shared(buf, + sample, params, settings); }); factory.registerOutputFormatProcessor("JSONStringsEachRowWithProgress", []( WriteBuffer & buf, const Block & sample, const RowOutputFormatParams & params, - const FormatSettings & format_settings) + const FormatSettings & _format_settings) { - return std::make_shared(buf, sample, params, format_settings, true); + FormatSettings settings = _format_settings; + settings.json.serialize_as_strings = true; + return std::make_shared(buf, + sample, params, settings); }); } diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp index 517f126060f..87eb88720c2 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp @@ -1,12 +1,45 @@ +#include + +#include +#include +#include +#include #include #include -#include #include namespace DB { +void JSONRowOutputFormat::addColumn(String name, DataTypePtr type, + bool & need_validate_utf8, std::string tabs) +{ + if (!type->textCanContainOnlyValidUTF8()) + need_validate_utf8 = true; + + WriteBufferFromOwnString buf; + writeJSONString(name, buf, settings); + + const auto * as_tuple = typeid_cast(type.get()); + const bool recurse = settings.json.named_tuple_as_object + && as_tuple && as_tuple->haveExplicitNames(); + + fields.emplace_back(FieldInfo{buf.str(), type, recurse, tabs}); + + if (recurse) + { + const auto & types = as_tuple->getElements(); + const auto & names = as_tuple->getElementNames(); + + assert(types.size() == names.size()); + for (size_t i = 0; i < types.size(); i++) + { + addColumn(names[i], types[i], need_validate_utf8, tabs + "\t"); + } + } +} + JSONRowOutputFormat::JSONRowOutputFormat( WriteBuffer & out_, const Block & header, @@ -17,20 +50,22 @@ JSONRowOutputFormat::JSONRowOutputFormat( { const auto & sample = getPort(PortKind::Main).getHeader(); NamesAndTypesList columns(sample.getNamesAndTypesList()); - fields.assign(columns.begin(), columns.end()); + fields.reserve(columns.size()); + + const std::string initial_tabs = settings.json.write_metadata ? "\t\t\t" : "\t\t"; bool need_validate_utf8 = false; - for (size_t i = 0; i < sample.columns(); ++i) + for (const auto & column : columns) { - if (!sample.getByPosition(i).type->textCanContainOnlyValidUTF8()) - need_validate_utf8 = true; - - WriteBufferFromOwnString buf; - writeJSONString(fields[i].name, buf, settings); - - fields[i].name = buf.str(); + addColumn(column.name, column.type, need_validate_utf8, initial_tabs); } +// for (size_t i = 0; i < fields.size(); i++) +// { +// fmt::print(stderr, "{}: '{}' '{}' '{}\n", +// i, fields[i].name, fields[i].type->getName(), fields[i].recurse); +// } + if (need_validate_utf8) { validating_ostr = std::make_unique(out); @@ -43,40 +78,76 @@ JSONRowOutputFormat::JSONRowOutputFormat( void JSONRowOutputFormat::writePrefix() { - writeCString("{\n", *ostr); - writeCString("\t\"meta\":\n", *ostr); - writeCString("\t[\n", *ostr); - - for (size_t i = 0; i < fields.size(); ++i) + if (settings.json.write_metadata) { - writeCString("\t\t{\n", *ostr); + writeCString("{\n", *ostr); + writeCString("\t\"meta\":\n", *ostr); + writeCString("\t[\n", *ostr); - writeCString("\t\t\t\"name\": ", *ostr); - writeString(fields[i].name, *ostr); - writeCString(",\n", *ostr); - writeCString("\t\t\t\"type\": ", *ostr); - writeJSONString(fields[i].type->getName(), *ostr, settings); - writeChar('\n', *ostr); + for (size_t i = 0; i < fields.size(); ++i) + { + writeCString("\t\t{\n", *ostr); - writeCString("\t\t}", *ostr); - if (i + 1 < fields.size()) - writeChar(',', *ostr); + writeCString("\t\t\t\"name\": ", *ostr); + writeString(fields[i].name, *ostr); + writeCString(",\n", *ostr); + writeCString("\t\t\t\"type\": ", *ostr); + writeJSONString(fields[i].type->getName(), *ostr, settings); + writeChar('\n', *ostr); + + writeCString("\t\t}", *ostr); + if (i + 1 < fields.size()) + writeChar(',', *ostr); + writeChar('\n', *ostr); + } + + writeCString("\t],\n", *ostr); writeChar('\n', *ostr); + writeCString("\t\"data\":\n", *ostr); + writeCString("\t", *ostr); } - - writeCString("\t],\n", *ostr); - writeChar('\n', *ostr); - writeCString("\t\"data\":\n", *ostr); - writeCString("\t[\n", *ostr); + writeCString("[\n", *ostr); } - void JSONRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) { - writeCString("\t\t\t", *ostr); +// fmt::print(stderr, "write field column '{}' type '{}'\n", +// column.getName(), type.getName()); + + writeString(fields[field_number].tabs, *ostr); writeString(fields[field_number].name, *ostr); writeCString(": ", *ostr); + // Sanity check: the input column type is the same as in header block. + // If I don't write out the raw pointer explicitly, for some reason clang + // complains about side effect in dereferencing the pointer: + // src/Processors/Formats/Impl/JSONRowOutputFormat.cpp:120:35: warning: expression with side effects will be evaluated despite being used as an operand to 'typeid' [-Wpotentially-evaluated-expression] + [[maybe_unused]] const IDataType * raw_ptr = fields[field_number].type.get(); + assert(typeid(type) == typeid(*raw_ptr)); + + if (fields[field_number].recurse) + { + const auto & tabs = fields[field_number].tabs; + ++field_number; + const auto & tuple_column = assert_cast(column); + const auto & nested_columns = tuple_column.getColumns(); + writeCString("{\n", *ostr); + for (size_t i = 0; i < nested_columns.size(); i++) + { + // field_number is incremented inside, and should match the nested + // columns. + writeField(*nested_columns[i], *fields[field_number].type, row_num); + if (i + 1 < nested_columns.size()) + { + writeCString(",", *ostr); + } + writeCString("\n", *ostr); + } + writeString(tabs, *ostr); + writeCString("}", *ostr); + return; + } + if (yield_strings) { WriteBufferFromOwnString buf; @@ -144,6 +215,12 @@ void JSONRowOutputFormat::writeSuffix() void JSONRowOutputFormat::writeBeforeTotals() { + if (!settings.json.write_metadata) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot output totals in JSON format without metadata"); + } + writeCString(",\n", *ostr); writeChar('\n', *ostr); writeCString("\t\"totals\":\n", *ostr); @@ -172,6 +249,12 @@ void JSONRowOutputFormat::writeAfterTotals() void JSONRowOutputFormat::writeBeforeExtremes() { + if (!settings.json.write_metadata) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot output extremes in JSON format without metadata"); + } + writeCString(",\n", *ostr); writeChar('\n', *ostr); writeCString("\t\"extremes\":\n", *ostr); @@ -217,17 +300,20 @@ void JSONRowOutputFormat::writeAfterExtremes() void JSONRowOutputFormat::writeLastSuffix() { - writeCString(",\n\n", *ostr); - writeCString("\t\"rows\": ", *ostr); - writeIntText(row_count, *ostr); + if (settings.json.write_metadata) + { + writeCString(",\n\n", *ostr); + writeCString("\t\"rows\": ", *ostr); + writeIntText(row_count, *ostr); - writeRowsBeforeLimitAtLeast(); + writeRowsBeforeLimitAtLeast(); - if (settings.write_statistics) - writeStatistics(); + if (settings.write_statistics) + writeStatistics(); - writeChar('\n', *ostr); - writeCString("}\n", *ostr); + writeChar('\n', *ostr); + writeCString("}\n", *ostr); + } ostr->next(); } diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.h b/src/Processors/Formats/Impl/JSONRowOutputFormat.h index 88b74afbabd..a4593663aeb 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.h @@ -70,6 +70,8 @@ protected: void writeRowsBeforeLimitAtLeast(); void writeStatistics(); + void addColumn(String name, DataTypePtr type, bool & need_validate_utf8, + std::string tabs); std::unique_ptr validating_ostr; /// Validates UTF-8 sequences, replaces bad sequences with replacement character. WriteBuffer * ostr; @@ -78,7 +80,16 @@ protected: size_t row_count = 0; bool applied_limit = false; size_t rows_before_limit = 0; - NamesAndTypes fields; + + struct FieldInfo + { + String name; + DataTypePtr type; + bool recurse = false; + std::string tabs; + }; + + std::vector fields; Progress progress; Stopwatch watch; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 8c7cd7b63d7..956e88f6641 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -166,6 +166,19 @@ StorageFile::StorageFile(int table_fd_, CommonArguments args) StorageFile::StorageFile(const std::string & table_path_, const std::string & user_files_path, CommonArguments args) : StorageFile(args) { + fmt::print(stderr, "Create storage file from file at \n{}\n", StackTrace().toString()); + + const auto & changes = args.context.getSettings().changes(); + for (const auto & change : changes) + { + fmt::print(stderr, "Changed setting '{}' to '{}'\n", + change.name, toString(change.value)); + } + + fmt::print(stderr, "delimiter = {}\n", + toString(args.context.getSettings().get("format_csv_delimiter"))); + + is_db_table = false; paths = getPathsList(table_path_, user_files_path, args.context); From 1a3fe377a6ce6196e139d5ceb49d73b74f9501f4 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 30 Oct 2020 04:52:37 +0300 Subject: [PATCH 010/258] prepare for format settings in File engine --- src/Formats/FormatFactory.cpp | 85 +++++--- src/Formats/FormatFactory.h | 15 +- src/Formats/FormatSettings.h | 190 +++++++++--------- src/Processors/Formats/IRowInputFormat.cpp | 2 - src/Processors/Formats/IRowInputFormat.h | 3 - src/Processors/Formats/IRowOutputFormat.h | 8 - .../Formats/Impl/ProtobufRowOutputFormat.cpp | 32 +-- .../Formats/Impl/ProtobufRowOutputFormat.h | 4 +- .../Formats/Impl/ValuesBlockInputFormat.cpp | 2 - src/Storages/Kafka/KafkaBlockOutputStream.cpp | 17 +- .../RabbitMQ/RabbitMQBlockOutputStream.cpp | 17 +- src/Storages/StorageFile.cpp | 52 +++-- src/Storages/StorageFile.h | 6 +- 13 files changed, 232 insertions(+), 201 deletions(-) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index d7601166136..c7c3c6c595a 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -41,8 +41,10 @@ const FormatFactory::Creators & FormatFactory::getCreators(const String & name) } -static FormatSettings getInputFormatSetting(const Settings & settings, const Context & context) +FormatSettings getInputFormatSettings(const Context & context) { + const auto & settings = context.getSettingsRef(); + FormatSettings format_settings; format_settings.csv.delimiter = settings.format_csv_delimiter; format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; @@ -92,8 +94,10 @@ static FormatSettings getInputFormatSetting(const Settings & settings, const Con return format_settings; } -static FormatSettings getOutputFormatSetting(const Settings & settings, const Context & context) +FormatSettings getOutputFormatSettings(const Context & context) { + const auto & settings = context.getSettingsRef(); + FormatSettings format_settings; format_settings.enable_streaming = settings.output_format_enable_streaming; format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; @@ -144,7 +148,7 @@ BlockInputStreamPtr FormatFactory::getInput( const Block & sample, const Context & context, UInt64 max_block_size, - ReadCallback callback) const + std::optional format_settings) const { if (name == "Native") return std::make_shared(buf, sample, 0); @@ -155,10 +159,12 @@ BlockInputStreamPtr FormatFactory::getInput( if (!input_getter) throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); - const Settings & settings = context.getSettingsRef(); - FormatSettings format_settings = getInputFormatSetting(settings, context); + if (!format_settings) + { + format_settings = getInputFormatSettings(context); + } - return input_getter(buf, sample, max_block_size, callback ? callback : ReadCallback(), format_settings); + return input_getter(buf, sample, max_block_size, {}, *format_settings); } const Settings & settings = context.getSettingsRef(); @@ -184,17 +190,21 @@ BlockInputStreamPtr FormatFactory::getInput( if (!input_getter) throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); - FormatSettings format_settings = getInputFormatSetting(settings, context); + if (!format_settings) + { + format_settings = getInputFormatSettings(context); + } RowInputFormatParams row_input_format_params; row_input_format_params.max_block_size = max_block_size; - row_input_format_params.allow_errors_num = format_settings.input_allow_errors_num; - row_input_format_params.allow_errors_ratio = format_settings.input_allow_errors_ratio; - row_input_format_params.callback = std::move(callback); + row_input_format_params.allow_errors_num = format_settings->input_allow_errors_num; + row_input_format_params.allow_errors_ratio = format_settings->input_allow_errors_ratio; row_input_format_params.max_execution_time = settings.max_execution_time; row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode; - auto input_creator_params = ParallelParsingBlockInputStream::InputCreatorParams{sample, row_input_format_params, format_settings}; + auto input_creator_params = + ParallelParsingBlockInputStream::InputCreatorParams{sample, + row_input_format_params, *format_settings}; ParallelParsingBlockInputStream::Params params{buf, input_getter, input_creator_params, file_segmentation_engine, static_cast(settings.max_threads), @@ -202,13 +212,15 @@ BlockInputStreamPtr FormatFactory::getInput( return std::make_shared(params); } - auto format = getInputFormat(name, buf, sample, context, max_block_size, std::move(callback)); + auto format = getInputFormat(name, buf, sample, context, max_block_size, + format_settings); return std::make_shared(std::move(format)); } -BlockOutputStreamPtr FormatFactory::getOutput( - const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, const bool ignore_no_row_delimiter) const +BlockOutputStreamPtr FormatFactory::getOutput(const String & name, + WriteBuffer & buf, const Block & sample, const Context & context, + WriteCallback callback, std::optional format_settings) const { if (!getCreators(name).output_processor_creator) { @@ -216,18 +228,23 @@ BlockOutputStreamPtr FormatFactory::getOutput( if (!output_getter) throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); - const Settings & settings = context.getSettingsRef(); - FormatSettings format_settings = getOutputFormatSetting(settings, context); + if (!format_settings) + { + format_settings = getOutputFormatSettings(context); + } /** Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. */ return std::make_shared( - output_getter(buf, sample, std::move(callback), format_settings), sample); + output_getter(buf, sample, std::move(callback), *format_settings), + sample); } - auto format = getOutputFormat(name, buf, sample, context, std::move(callback), ignore_no_row_delimiter); - return std::make_shared(std::make_shared(format), sample); + auto format = getOutputFormat(name, buf, sample, context, std::move(callback), + format_settings); + return std::make_shared( + std::make_shared(format), sample); } @@ -237,24 +254,27 @@ InputFormatPtr FormatFactory::getInputFormat( const Block & sample, const Context & context, UInt64 max_block_size, - ReadCallback callback) const + std::optional format_settings) const { const auto & input_getter = getCreators(name).input_processor_creator; if (!input_getter) throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT); const Settings & settings = context.getSettingsRef(); - FormatSettings format_settings = getInputFormatSetting(settings, context); + + if (!format_settings) + { + format_settings = getInputFormatSettings(context); + } RowInputFormatParams params; params.max_block_size = max_block_size; - params.allow_errors_num = format_settings.input_allow_errors_num; - params.allow_errors_ratio = format_settings.input_allow_errors_ratio; - params.callback = std::move(callback); + params.allow_errors_num = format_settings->input_allow_errors_num; + params.allow_errors_ratio = format_settings->input_allow_errors_ratio; params.max_execution_time = settings.max_execution_time; params.timeout_overflow_mode = settings.timeout_overflow_mode; - auto format = input_getter(buf, sample, params, format_settings); + auto format = input_getter(buf, sample, params, *format_settings); /// It's a kludge. Because I cannot remove context from values format. if (auto * values = typeid_cast(format.get())) @@ -265,26 +285,29 @@ InputFormatPtr FormatFactory::getInputFormat( OutputFormatPtr FormatFactory::getOutputFormat( - const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback, const bool ignore_no_row_delimiter) const + const String & name, WriteBuffer & buf, const Block & sample, + const Context & context, WriteCallback callback, + std::optional format_settings) const { const auto & output_getter = getCreators(name).output_processor_creator; if (!output_getter) throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); - const Settings & settings = context.getSettingsRef(); - FormatSettings format_settings = getOutputFormatSetting(settings, context); + if (!format_settings) + { + format_settings = getOutputFormatSettings(context); + } RowOutputFormatParams params; - params.ignore_no_row_delimiter = ignore_no_row_delimiter; params.callback = std::move(callback); /** TODO: Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. */ - auto format = output_getter(buf, sample, params, format_settings); + auto format = output_getter(buf, sample, params, *format_settings); /// Enable auto-flush for streaming mode. Currently it is needed by INSERT WATCH query. - if (format_settings.enable_streaming) + if (format_settings->enable_streaming) format->setAutoFlush(); /// It's a kludge. Because I cannot remove context from MySQL format. diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index d49414e3944..78d4d61ea09 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -16,6 +17,7 @@ namespace DB class Block; class Context; struct FormatSettings; +struct Settings; class ReadBuffer; class WriteBuffer; @@ -32,6 +34,8 @@ struct RowOutputFormatParams; using InputFormatPtr = std::shared_ptr; using OutputFormatPtr = std::shared_ptr; +FormatSettings getInputFormatSettings(const Context & context); +FormatSettings getOutputFormatSettings(const Context & context); /** Allows to create an IBlockInputStream or IBlockOutputStream by the name of the format. * Note: format and compression are independent things. @@ -105,10 +109,11 @@ public: const Block & sample, const Context & context, UInt64 max_block_size, - ReadCallback callback = {}) const; + std::optional format_settings = std::nullopt) const; BlockOutputStreamPtr getOutput(const String & name, WriteBuffer & buf, - const Block & sample, const Context & context, WriteCallback callback = {}, const bool ignore_no_row_delimiter = false) const; + const Block & sample, const Context & context, WriteCallback callback = {}, + std::optional format_settings = std::nullopt) const; InputFormatPtr getInputFormat( const String & name, @@ -116,10 +121,12 @@ public: const Block & sample, const Context & context, UInt64 max_block_size, - ReadCallback callback = {}) const; + std::optional format_settings = std::nullopt) const; OutputFormatPtr getOutputFormat( - const String & name, WriteBuffer & buf, const Block & sample, const Context & context, WriteCallback callback = {}, const bool ignore_no_row_delimiter = false) const; + const String & name, WriteBuffer & buf, const Block & sample, + const Context & context, WriteCallback callback = {}, + std::optional format_settings = std::nullopt) const; /// Register format by its name. void registerInputFormat(const String & name, InputCreator input_creator); diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 102ae4a5a24..76dc7bc1729 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -17,80 +17,6 @@ struct FormatSettings /// Option means that each chunk of data need to be formatted independently. Also each chunk will be flushed at the end of processing. bool enable_streaming = false; - struct JSON - { - bool quote_64bit_integers = true; - bool quote_denormals = true; - bool escape_forward_slashes = true; - bool write_metadata = false; - bool named_tuple_as_object = true; - bool list_of_rows = false; - bool serialize_as_strings = false; - }; - - JSON json; - - struct CSV - { - char delimiter = ','; - bool allow_single_quotes = true; - bool allow_double_quotes = true; - bool unquoted_null_literal_as_null = false; - bool empty_as_default = false; - bool crlf_end_of_line = false; - bool input_format_enum_as_number = false; - }; - - CSV csv; - - struct Pretty - { - UInt64 max_rows = 10000; - UInt64 max_column_pad_width = 250; - UInt64 max_value_width = 10000; - bool color = true; - - bool output_format_pretty_row_numbers = false; - - enum class Charset - { - UTF8, - ASCII, - }; - - Charset charset = Charset::UTF8; - }; - - Pretty pretty; - - struct Values - { - bool interpret_expressions = true; - bool deduce_templates_of_expressions = true; - bool accurate_types_of_literals = true; - }; - - Values values; - - struct Template - { - String resultset_format; - String row_format; - String row_between_delimiter; - }; - - Template template_settings; - - struct TSV - { - bool empty_as_default = false; - bool crlf_end_of_line = false; - String null_representation = "\\N"; - bool input_format_enum_as_number = false; - }; - - TSV tsv; - bool skip_unknown_fields = false; bool with_names_use_header = false; bool write_statistics = true; @@ -117,24 +43,29 @@ struct FormatSettings UInt64 input_allow_errors_num = 0; Float32 input_allow_errors_ratio = 0; - struct Arrow + struct { UInt64 row_group_size = 1000000; } arrow; - struct Parquet + struct { - UInt64 row_group_size = 1000000; - } parquet; + String schema_registry_url; + String output_codec; + UInt64 output_sync_interval = 16 * 1024; + bool allow_missing_fields = false; + } avro; - struct Schema + struct CSV { - std::string format_schema; - std::string format_schema_path; - bool is_server = false; - }; - - Schema schema; + char delimiter = ','; + bool allow_single_quotes = true; + bool allow_double_quotes = true; + bool unquoted_null_literal_as_null = false; + bool empty_as_default = false; + bool crlf_end_of_line = false; + bool input_format_enum_as_number = false; + } csv; struct Custom { @@ -145,29 +76,92 @@ struct FormatSettings std::string row_between_delimiter; std::string field_delimiter; std::string escaping_rule; - }; + } custom; - Custom custom; - - struct Avro + struct { - String schema_registry_url; - String output_codec; - UInt64 output_sync_interval = 16 * 1024; - bool allow_missing_fields = false; - }; + bool quote_64bit_integers = true; + bool quote_denormals = true; + bool escape_forward_slashes = true; + bool write_metadata = false; + bool named_tuple_as_object = true; + bool list_of_rows = false; + bool serialize_as_strings = false; + } json; - Avro avro; + struct + { + UInt64 row_group_size = 1000000; + } parquet; - struct Regexp + struct Pretty + { + UInt64 max_rows = 10000; + UInt64 max_column_pad_width = 250; + UInt64 max_value_width = 10000; + bool color = true; + + bool output_format_pretty_row_numbers = false; + + enum class Charset + { + UTF8, + ASCII, + }; + + Charset charset = Charset::UTF8; + } pretty; + + struct + { + bool write_row_delimiters = true; + /** + * Some buffers (kafka / rabbit) split the rows internally using callback + * so we can push there formats without framing / delimiters (like + * ProtobufSingle). In other cases you can't write more than single row + * in unframed format. + * Not sure we need this parameter at all, it only serves as an additional + * safety check in ProtobufSingle format, but exporting constant-size + * records w/o delimiters might be generally useful, not only for Kafka. + */ + bool allow_many_rows_no_delimiters = false; + } protobuf; + + struct { std::string regexp; std::string escaping_rule; bool skip_unmatched = false; - }; + } regexp; - Regexp regexp; + struct + { + std::string format_schema; + std::string format_schema_path; + bool is_server = false; + } schema; + struct + { + String resultset_format; + String row_format; + String row_between_delimiter; + } template_settings; + + struct + { + bool empty_as_default = false; + bool crlf_end_of_line = false; + String null_representation = "\\N"; + bool input_format_enum_as_number = false; + } tsv; + + struct + { + bool interpret_expressions = true; + bool deduce_templates_of_expressions = true; + bool accurate_types_of_literals = true; + } values; }; } diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index 12d4db1f4a8..48cfdb12d8b 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -63,8 +63,6 @@ Chunk IRowInputFormat::generate() info.read_columns.clear(); if (!readRow(columns, info)) break; - if (params.callback) - params.callback(); for (size_t column_idx = 0; column_idx < info.read_columns.size(); ++column_idx) { diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index 1931fba2a0d..14109f9c2be 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -27,9 +27,6 @@ struct RowInputFormatParams UInt64 allow_errors_num; Float64 allow_errors_ratio; - using ReadCallback = std::function; - ReadCallback callback; - Poco::Timespan max_execution_time = 0; OverflowMode timeout_overflow_mode = OverflowMode::THROW; }; diff --git a/src/Processors/Formats/IRowOutputFormat.h b/src/Processors/Formats/IRowOutputFormat.h index 4312691ea5e..4fb94f7b7f7 100644 --- a/src/Processors/Formats/IRowOutputFormat.h +++ b/src/Processors/Formats/IRowOutputFormat.h @@ -15,14 +15,6 @@ struct RowOutputFormatParams // Callback used to indicate that another row is written. WriteCallback callback; - - /** - * some buffers (kafka / rabbit) split the rows internally using callback - * so we can push there formats without framing / delimiters - * (like ProtobufSingle). In other cases you can't write more than single row - * in unframed format. - */ - bool ignore_no_row_delimiter = false; }; class WriteBuffer; diff --git a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp index 930a83c52da..3c885e80e31 100644 --- a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp @@ -23,18 +23,22 @@ ProtobufRowOutputFormat::ProtobufRowOutputFormat( const Block & header, const RowOutputFormatParams & params_, const FormatSchemaInfo & format_schema, - const bool use_length_delimiters_) + const FormatSettings & settings) : IRowOutputFormat(header, out_, params_) , data_types(header.getDataTypes()) - , writer(out, ProtobufSchemas::instance().getMessageTypeForFormatSchema(format_schema), header.getNames(), use_length_delimiters_) - , throw_on_multiple_rows_undelimited(!use_length_delimiters_ && !params_.ignore_no_row_delimiter) + , writer(out, + ProtobufSchemas::instance().getMessageTypeForFormatSchema(format_schema), + header.getNames(), settings.protobuf.write_row_delimiters) + , allow_only_one_row( + !settings.protobuf.write_row_delimiters + && !settings.protobuf.allow_many_rows_no_delimiters) { value_indices.resize(header.columns()); } void ProtobufRowOutputFormat::write(const Columns & columns, size_t row_num) { - if (throw_on_multiple_rows_undelimited && !first_row) + if (allow_only_one_row && !first_row) { throw Exception("The ProtobufSingle format can't be used to write multiple rows because this format doesn't have any row delimiter.", ErrorCodes::NO_ROW_DELIMITER); } @@ -51,19 +55,23 @@ void ProtobufRowOutputFormat::write(const Columns & columns, size_t row_num) void registerOutputFormatProcessorProtobuf(FormatFactory & factory) { - for (bool use_length_delimiters : {false, true}) + for (bool write_row_delimiters : {false, true}) { factory.registerOutputFormatProcessor( - use_length_delimiters ? "Protobuf" : "ProtobufSingle", - [use_length_delimiters](WriteBuffer & buf, + write_row_delimiters ? "Protobuf" : "ProtobufSingle", + [write_row_delimiters](WriteBuffer & buf, const Block & header, const RowOutputFormatParams & params, - const FormatSettings & settings) + const FormatSettings & _settings) { - return std::make_shared(buf, header, params, - FormatSchemaInfo(settings.schema.format_schema, "Protobuf", true, - settings.schema.is_server, settings.schema.format_schema_path), - use_length_delimiters); + FormatSettings settings = _settings; + settings.protobuf.write_row_delimiters = write_row_delimiters; + return std::make_shared( + buf, header, params, + FormatSchemaInfo(settings.schema.format_schema, "Protobuf", + true, settings.schema.is_server, + settings.schema.format_schema_path), + settings); }); } } diff --git a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h index 740efcfa24c..847f7607ff5 100644 --- a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h @@ -41,7 +41,7 @@ public: const Block & header, const RowOutputFormatParams & params_, const FormatSchemaInfo & format_schema, - const bool use_length_delimiters_); + const FormatSettings & settings); String getName() const override { return "ProtobufRowOutputFormat"; } @@ -53,7 +53,7 @@ private: DataTypes data_types; ProtobufWriter writer; std::vector value_indices; - const bool throw_on_multiple_rows_undelimited; + const bool allow_only_one_row; }; } diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index de5a1b71580..c3b753e7261 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -54,8 +54,6 @@ Chunk ValuesBlockInputFormat::generate() if (buf.eof() || *buf.position() == ';') break; readRow(columns, rows_in_block); - if (params.callback) - params.callback(); } catch (Exception & e) { diff --git a/src/Storages/Kafka/KafkaBlockOutputStream.cpp b/src/Storages/Kafka/KafkaBlockOutputStream.cpp index 9d7fe465d44..685ab59fdac 100644 --- a/src/Storages/Kafka/KafkaBlockOutputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockOutputStream.cpp @@ -32,13 +32,16 @@ void KafkaBlockOutputStream::writePrefix() if (!buffer) throw Exception("Failed to create Kafka producer!", ErrorCodes::CANNOT_CREATE_IO_BUFFER); - child = FormatFactory::instance().getOutput( - storage.getFormatName(), *buffer, getHeader(), *context, [this](const Columns & columns, size_t row) - { - buffer->countRow(columns, row); - }, - /* ignore_no_row_delimiter = */ true - ); + auto format_settings = getOutputFormatSettings(*context); + format_settings.protobuf.allow_many_rows_no_delimiters = true; + + child = FormatFactory::instance().getOutput(storage.getFormatName(), *buffer, + getHeader(), *context, + [this](const Columns & columns, size_t row) + { + buffer->countRow(columns, row); + }, + format_settings); } void KafkaBlockOutputStream::write(const Block & block) diff --git a/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp b/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp index 76129dee30d..c4b2f187a5a 100644 --- a/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp +++ b/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp @@ -42,13 +42,16 @@ void RabbitMQBlockOutputStream::writePrefix() buffer->activateWriting(); - child = FormatFactory::instance().getOutput( - storage.getFormatName(), *buffer, getHeader(), context, [this](const Columns & /* columns */, size_t /* rows */) - { - buffer->countRow(); - }, - /* ignore_no_row_delimiter = */ true - ); + auto format_settings = getOutputFormatSettings(context); + format_settings.protobuf.allow_many_rows_no_delimiters = true; + + child = FormatFactory::instance().getOutput(storage.getFormatName(), *buffer, + getHeader(), context, + [this](const Columns & /* columns */, size_t /* rows */) + { + buffer->countRow(); + }, + format_settings); } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 956e88f6641..5a9fe7872e2 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -594,32 +594,40 @@ void StorageFile::truncate( void registerStorageFile(StorageFactory & factory) { + StorageFactory::StorageFeatures storage_features{ + .supports_settings = true, + .source_access_type = AccessType::FILE + }; + factory.registerStorage( "File", - [](const StorageFactory::Arguments & args) + [](const StorageFactory::Arguments & factory_args) { - ASTs & engine_args = args.engine_args; + StorageFile::CommonArguments storage_args{ + .table_id = factory_args.table_id, + .columns = factory_args.columns, + .constraints = factory_args.constraints, + .context = factory_args.context + }; - if (!(engine_args.size() >= 1 && engine_args.size() <= 3)) // NOLINT + ASTs & engine_args_ast = factory_args.engine_args; + + if (!(engine_args_ast.size() >= 1 && engine_args_ast.size() <= 3)) // NOLINT throw Exception( "Storage File requires from 1 to 3 arguments: name of used format, source and compression_method.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.local_context); - String format_name = engine_args[0]->as().value.safeGet(); + engine_args_ast[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args_ast[0], factory_args.local_context); + storage_args.format_name = engine_args_ast[0]->as().value.safeGet(); - String compression_method; - StorageFile::CommonArguments common_args{ - args.table_id, format_name, compression_method, args.columns, args.constraints, args.context}; - - if (engine_args.size() == 1) /// Table in database - return StorageFile::create(args.relative_data_path, common_args); + if (engine_args_ast.size() == 1) /// Table in database + return StorageFile::create(factory_args.relative_data_path, storage_args); /// Will use FD if engine_args[1] is int literal or identifier with std* name int source_fd = -1; String source_path; - if (auto opt_name = tryGetIdentifierName(engine_args[1])) + if (auto opt_name = tryGetIdentifierName(engine_args_ast[1])) { if (*opt_name == "stdin") source_fd = STDIN_FILENO; @@ -631,7 +639,7 @@ void registerStorageFile(StorageFactory & factory) throw Exception( "Unknown identifier '" + *opt_name + "' in second arg of File storage constructor", ErrorCodes::UNKNOWN_IDENTIFIER); } - else if (const auto * literal = engine_args[1]->as()) + else if (const auto * literal = engine_args_ast[1]->as()) { auto type = literal->value.getType(); if (type == Field::Types::Int64) @@ -644,23 +652,23 @@ void registerStorageFile(StorageFactory & factory) throw Exception("Second argument must be path or file descriptor", ErrorCodes::BAD_ARGUMENTS); } - if (engine_args.size() == 3) + if (engine_args_ast.size() == 3) { - engine_args[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[2], args.local_context); - compression_method = engine_args[2]->as().value.safeGet(); + engine_args_ast[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args_ast[2], factory_args.local_context); + storage_args.compression_method = engine_args_ast[2]->as().value.safeGet(); } else - compression_method = "auto"; + storage_args.compression_method = "auto"; if (0 <= source_fd) /// File descriptor - return StorageFile::create(source_fd, common_args); + return StorageFile::create(source_fd, storage_args); else /// User's file - return StorageFile::create(source_path, args.context.getUserFilesPath(), common_args); + return StorageFile::create(source_path, factory_args.context.getUserFilesPath(), storage_args); }, - { - .source_access_type = AccessType::FILE, - }); + storage_features); } + + NamesAndTypesList StorageFile::getVirtuals() const { return NamesAndTypesList{ diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index f331538b4c7..cc8f6dc2b8a 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -50,9 +50,9 @@ public: struct CommonArguments { - const StorageID & table_id; - const std::string & format_name; - const std::string & compression_method; + StorageID table_id; + std::string format_name; + std::string compression_method; const ColumnsDescription & columns; const ConstraintsDescription & constraints; const Context & context; From 145ad495ce9322bec999d939af85179980984ea2 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 30 Oct 2020 13:42:32 +0300 Subject: [PATCH 011/258] fixes --- src/Formats/tests/tab_separated_streams.cpp | 4 ++-- src/Functions/tuple.cpp | 8 ++++++-- .../Formats/Impl/JSONRowOutputFormat.cpp | 15 ++++++++++----- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/Formats/tests/tab_separated_streams.cpp b/src/Formats/tests/tab_separated_streams.cpp index bb38d51cf98..bd733e4b9aa 100644 --- a/src/Formats/tests/tab_separated_streams.cpp +++ b/src/Formats/tests/tab_separated_streams.cpp @@ -38,8 +38,8 @@ try FormatSettings format_settings; - RowInputFormatParams in_params{DEFAULT_INSERT_BLOCK_SIZE, 0, 0, []{}}; - RowOutputFormatParams out_params{[](const Columns & /* columns */, size_t /* row */){},false}; + RowInputFormatParams in_params{DEFAULT_INSERT_BLOCK_SIZE, 0, 0}; + RowOutputFormatParams out_params{[](const Columns & /* columns */, size_t /* row */){}}; InputFormatPtr input_format = std::make_shared(sample, in_buf, in_params, false, false, format_settings); BlockInputStreamPtr block_input = std::make_shared(std::move(input_format)); diff --git a/src/Functions/tuple.cpp b/src/Functions/tuple.cpp index 6522723dde4..e4cbde6f9d3 100644 --- a/src/Functions/tuple.cpp +++ b/src/Functions/tuple.cpp @@ -66,9 +66,13 @@ public: names.emplace_back(argument.name); } - /// Create named tuple if possible. + /// Create named tuple if possible. We don't print tuple element names + /// because they are bad anyway -- aliases are not used, e.g. tuple(1 a) + /// will have element name '1' and not 'a'. If we ever change this, and + /// add the ability to access tuple elements by name, like tuple(1 a).a, + /// we should probably enable printing for better discoverability. if (DataTypeTuple::canBeCreatedWithNames(names)) - return std::make_shared(types, names); + return std::make_shared(types, names, false /*print names*/); return std::make_shared(types); } diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp index 87eb88720c2..b7514bcff89 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp @@ -12,6 +12,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + void JSONRowOutputFormat::addColumn(String name, DataTypePtr type, bool & need_validate_utf8, std::string tabs) { @@ -29,13 +34,13 @@ void JSONRowOutputFormat::addColumn(String name, DataTypePtr type, if (recurse) { - const auto & types = as_tuple->getElements(); + const auto & element_types = as_tuple->getElements(); const auto & names = as_tuple->getElementNames(); - assert(types.size() == names.size()); - for (size_t i = 0; i < types.size(); i++) + assert(element_types.size() == names.size()); + for (size_t i = 0; i < element_types.size(); i++) { - addColumn(names[i], types[i], need_validate_utf8, tabs + "\t"); + addColumn(names[i], element_types[i], need_validate_utf8, tabs + "\t"); } } } @@ -195,7 +200,7 @@ void JSONRowOutputFormat::writeRowStartDelimiter() void JSONRowOutputFormat::writeRowEndDelimiter() { writeChar('\n', *ostr); - writeCString("\t\t}", *ostr); + writeCString("\t\t}\n", *ostr); field_number = 0; ++row_count; } From e81140b5fadb7382ef2bdb91b15212acb555ec04 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 30 Oct 2020 15:13:53 +0300 Subject: [PATCH 012/258] format settings in File engine --- .../AggregateFunctionSumMap.h | 2 +- src/Formats/FormatFactory.cpp | 136 +++++++----------- src/Formats/FormatFactory.h | 3 +- src/Formats/FormatSettings.h | 2 +- .../Impl/JSONEachRowRowOutputFormat.cpp | 13 +- .../Formats/Impl/JSONRowOutputFormat.cpp | 2 +- src/Storages/Kafka/KafkaBlockOutputStream.cpp | 2 +- .../RabbitMQ/RabbitMQBlockOutputStream.cpp | 2 +- src/Storages/StorageFile.cpp | 59 ++++++-- src/Storages/StorageFile.h | 2 + src/TableFunctions/TableFunctionFile.cpp | 23 ++- 11 files changed, 131 insertions(+), 115 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index 6533903bbc1..456334ee9c3 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -105,7 +105,7 @@ public: types.emplace_back(std::make_shared(result_type)); } - return std::make_shared(types, Strings{"keys", "values"}); + return std::make_shared(types); } static const auto & getArgumentColumns(const IColumn**& columns) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index c7c3c6c595a..814b75960fe 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -41,102 +41,76 @@ const FormatFactory::Creators & FormatFactory::getCreators(const String & name) } -FormatSettings getInputFormatSettings(const Context & context) +FormatSettings getFormatSettings(const Context & context) { const auto & settings = context.getSettingsRef(); FormatSettings format_settings; - format_settings.csv.delimiter = settings.format_csv_delimiter; - format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; + + format_settings.avro.allow_missing_fields = settings.input_format_avro_allow_missing_fields; + format_settings.avro.output_codec = settings.output_format_avro_codec; + format_settings.avro.output_sync_interval = settings.output_format_avro_sync_interval; + format_settings.avro.schema_registry_url = settings.format_avro_schema_registry_url.toString(); format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; - format_settings.csv.unquoted_null_literal_as_null = settings.input_format_csv_unquoted_null_literal_as_null; + format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; + format_settings.csv.crlf_end_of_line = settings.output_format_csv_crlf_end_of_line; + format_settings.csv.delimiter = settings.format_csv_delimiter; format_settings.csv.empty_as_default = settings.input_format_defaults_for_omitted_fields; format_settings.csv.input_format_enum_as_number = settings.input_format_csv_enum_as_number; - format_settings.null_as_default = settings.input_format_null_as_default; - format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; - format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions; - format_settings.values.accurate_types_of_literals = settings.input_format_values_accurate_types_of_literals; - format_settings.with_names_use_header = settings.input_format_with_names_use_header; - format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields; - format_settings.import_nested_json = settings.input_format_import_nested_json; + format_settings.csv.unquoted_null_literal_as_null = settings.input_format_csv_unquoted_null_literal_as_null; + format_settings.custom.escaping_rule = settings.format_custom_escaping_rule; + format_settings.custom.field_delimiter = settings.format_custom_field_delimiter; + format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter; + format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter; + format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter; + format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; + format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; + format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; format_settings.date_time_input_format = settings.date_time_input_format; + format_settings.date_time_output_format = settings.date_time_output_format; + format_settings.enable_streaming = settings.output_format_enable_streaming; + format_settings.import_nested_json = settings.input_format_import_nested_json; format_settings.input_allow_errors_num = settings.input_format_allow_errors_num; format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio; - format_settings.template_settings.resultset_format = settings.format_template_resultset; - format_settings.template_settings.row_format = settings.format_template_row; - format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter; - format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default; - format_settings.tsv.input_format_enum_as_number = settings.input_format_tsv_enum_as_number; + format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes; + format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; + format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; + format_settings.json.write_metadata = settings.output_format_json_write_metadata; + format_settings.null_as_default = settings.input_format_null_as_default; + format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size; + format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; + format_settings.pretty.color = settings.output_format_pretty_color; + format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; + format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; + format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; + format_settings.pretty.output_format_pretty_row_numbers = settings.output_format_pretty_row_numbers; + format_settings.regexp.escaping_rule = settings.format_regexp_escaping_rule; + format_settings.regexp.regexp = settings.format_regexp; + format_settings.regexp.skip_unmatched = settings.format_regexp_skip_unmatched; format_settings.schema.format_schema = settings.format_schema; format_settings.schema.format_schema_path = context.getFormatSchemaPath(); format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER); - format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter; - format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter; - format_settings.custom.escaping_rule = settings.format_custom_escaping_rule; - format_settings.custom.field_delimiter = settings.format_custom_field_delimiter; - format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; - format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; - format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; - format_settings.regexp.regexp = settings.format_regexp; - format_settings.regexp.escaping_rule = settings.format_regexp_escaping_rule; - format_settings.regexp.skip_unmatched = settings.format_regexp_skip_unmatched; + format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields; + format_settings.template_settings.resultset_format = settings.format_template_resultset; + format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter; + format_settings.template_settings.row_format = settings.format_template_row; + format_settings.tsv.crlf_end_of_line = settings.output_format_tsv_crlf_end_of_line; + format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default; + format_settings.tsv.input_format_enum_as_number = settings.input_format_tsv_enum_as_number; + format_settings.tsv.null_representation = settings.output_format_tsv_null_representation; + format_settings.values.accurate_types_of_literals = settings.input_format_values_accurate_types_of_literals; + format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions; + format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; + format_settings.with_names_use_header = settings.input_format_with_names_use_header; + format_settings.write_statistics = settings.output_format_write_statistics; /// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context - if (context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER)) + if (format_settings.schema.is_server) { const Poco::URI & avro_schema_registry_url = settings.format_avro_schema_registry_url; if (!avro_schema_registry_url.empty()) context.getRemoteHostFilter().checkURL(avro_schema_registry_url); } - format_settings.avro.schema_registry_url = settings.format_avro_schema_registry_url.toString(); - format_settings.avro.allow_missing_fields = settings.input_format_avro_allow_missing_fields; - - return format_settings; -} - -FormatSettings getOutputFormatSettings(const Context & context) -{ - const auto & settings = context.getSettingsRef(); - - FormatSettings format_settings; - format_settings.enable_streaming = settings.output_format_enable_streaming; - format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; - format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; - format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes; - format_settings.json.write_metadata = settings.output_format_json_write_metadata; - //format_settings.json.named_tuple_as_object = settings.output_format_json_named_tuple_as_object; - format_settings.csv.delimiter = settings.format_csv_delimiter; - format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; - format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; - format_settings.csv.crlf_end_of_line = settings.output_format_csv_crlf_end_of_line; - format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; - format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; - format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; - format_settings.pretty.color = settings.output_format_pretty_color; - format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? - FormatSettings::Pretty::Charset::ASCII : - FormatSettings::Pretty::Charset::UTF8; - format_settings.pretty.output_format_pretty_row_numbers = settings.output_format_pretty_row_numbers; - format_settings.template_settings.resultset_format = settings.format_template_resultset; - format_settings.template_settings.row_format = settings.format_template_row; - format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter; - format_settings.tsv.crlf_end_of_line = settings.output_format_tsv_crlf_end_of_line; - format_settings.tsv.null_representation = settings.output_format_tsv_null_representation; - format_settings.write_statistics = settings.output_format_write_statistics; - format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size; - format_settings.schema.format_schema = settings.format_schema; - format_settings.schema.format_schema_path = context.getFormatSchemaPath(); - format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER); - format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter; - format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter; - format_settings.custom.escaping_rule = settings.format_custom_escaping_rule; - format_settings.custom.field_delimiter = settings.format_custom_field_delimiter; - format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; - format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; - format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; - format_settings.avro.output_codec = settings.output_format_avro_codec; - format_settings.avro.output_sync_interval = settings.output_format_avro_sync_interval; - format_settings.date_time_output_format = settings.date_time_output_format; return format_settings; } @@ -161,7 +135,7 @@ BlockInputStreamPtr FormatFactory::getInput( if (!format_settings) { - format_settings = getInputFormatSettings(context); + format_settings = getFormatSettings(context); } return input_getter(buf, sample, max_block_size, {}, *format_settings); @@ -192,7 +166,7 @@ BlockInputStreamPtr FormatFactory::getInput( if (!format_settings) { - format_settings = getInputFormatSettings(context); + format_settings = getFormatSettings(context); } RowInputFormatParams row_input_format_params; @@ -230,7 +204,7 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, if (!format_settings) { - format_settings = getOutputFormatSettings(context); + format_settings = getFormatSettings(context); } /** Materialization is needed, because formats can use the functions `IDataType`, @@ -264,7 +238,7 @@ InputFormatPtr FormatFactory::getInputFormat( if (!format_settings) { - format_settings = getInputFormatSettings(context); + format_settings = getFormatSettings(context); } RowInputFormatParams params; @@ -295,7 +269,7 @@ OutputFormatPtr FormatFactory::getOutputFormat( if (!format_settings) { - format_settings = getOutputFormatSettings(context); + format_settings = getFormatSettings(context); } RowOutputFormatParams params; diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 78d4d61ea09..619acd10e0f 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -34,8 +34,7 @@ struct RowOutputFormatParams; using InputFormatPtr = std::shared_ptr; using OutputFormatPtr = std::shared_ptr; -FormatSettings getInputFormatSettings(const Context & context); -FormatSettings getOutputFormatSettings(const Context & context); +FormatSettings getFormatSettings(const Context & context); /** Allows to create an IBlockInputStream or IBlockOutputStream by the name of the format. * Note: format and compression are independent things. diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 76dc7bc1729..a4f2962c223 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -85,7 +85,7 @@ struct FormatSettings bool escape_forward_slashes = true; bool write_metadata = false; bool named_tuple_as_object = true; - bool list_of_rows = false; + bool array_of_rows = false; bool serialize_as_strings = false; } json; diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp index 2459c6f2451..79027b4afac 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp @@ -68,7 +68,7 @@ void JSONEachRowRowOutputFormat::writeRowEndDelimiter() void JSONEachRowRowOutputFormat::writeRowBetweenDelimiter() { - if (settings.json.list_of_rows) + if (settings.json.array_of_rows) { writeCString(",\n", out); } @@ -81,7 +81,7 @@ void JSONEachRowRowOutputFormat::writeRowBetweenDelimiter() void JSONEachRowRowOutputFormat::writePrefix() { - if (settings.json.list_of_rows) + if (settings.json.array_of_rows) { writeCString("[\n", out); } @@ -90,9 +90,10 @@ void JSONEachRowRowOutputFormat::writePrefix() void JSONEachRowRowOutputFormat::writeSuffix() { - if (settings.json.list_of_rows) + writeCString("\n"); + if (settings.json.array_of_rows) { - writeCString("\n]\n", out); + writeCString("]\n", out); } } @@ -123,14 +124,14 @@ void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory) settings); }); - factory.registerOutputFormatProcessor("JSONList", []( + factory.registerOutputFormatProcessor("JSONArray", []( WriteBuffer & buf, const Block & sample, const RowOutputFormatParams & params, const FormatSettings & _format_settings) { FormatSettings settings = _format_settings; - settings.json.list_of_rows = true; + settings.json.array_of_rows = true; return std::make_shared(buf, sample, params, settings); }); diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp index b7514bcff89..babb217ea15 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp @@ -200,7 +200,7 @@ void JSONRowOutputFormat::writeRowStartDelimiter() void JSONRowOutputFormat::writeRowEndDelimiter() { writeChar('\n', *ostr); - writeCString("\t\t}\n", *ostr); + writeCString("\t\t}", *ostr); field_number = 0; ++row_count; } diff --git a/src/Storages/Kafka/KafkaBlockOutputStream.cpp b/src/Storages/Kafka/KafkaBlockOutputStream.cpp index 685ab59fdac..e7bf562339f 100644 --- a/src/Storages/Kafka/KafkaBlockOutputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockOutputStream.cpp @@ -32,7 +32,7 @@ void KafkaBlockOutputStream::writePrefix() if (!buffer) throw Exception("Failed to create Kafka producer!", ErrorCodes::CANNOT_CREATE_IO_BUFFER); - auto format_settings = getOutputFormatSettings(*context); + auto format_settings = getFormatSettings(*context); format_settings.protobuf.allow_many_rows_no_delimiters = true; child = FormatFactory::instance().getOutput(storage.getFormatName(), *buffer, diff --git a/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp b/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp index c4b2f187a5a..b3bd57bdd0b 100644 --- a/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp +++ b/src/Storages/RabbitMQ/RabbitMQBlockOutputStream.cpp @@ -42,7 +42,7 @@ void RabbitMQBlockOutputStream::writePrefix() buffer->activateWriting(); - auto format_settings = getOutputFormatSettings(context); + auto format_settings = getFormatSettings(context); format_settings.protobuf.allow_many_rows_no_delimiters = true; child = FormatFactory::instance().getOutput(storage.getFormatName(), *buffer, diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 5a9fe7872e2..d175736e8f8 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -166,7 +167,8 @@ StorageFile::StorageFile(int table_fd_, CommonArguments args) StorageFile::StorageFile(const std::string & table_path_, const std::string & user_files_path, CommonArguments args) : StorageFile(args) { - fmt::print(stderr, "Create storage file from file at \n{}\n", StackTrace().toString()); + fmt::print(stderr, "Create storage File '{}' from file at \n{}\n", + args.table_id.getNameForLogs(), StackTrace().toString()); const auto & changes = args.context.getSettings().changes(); for (const auto & change : changes) @@ -175,7 +177,7 @@ StorageFile::StorageFile(const std::string & table_path_, const std::string & us change.name, toString(change.value)); } - fmt::print(stderr, "delimiter = {}\n", + fmt::print(stderr, "delimiter = '{}'\n", toString(args.context.getSettings().get("format_csv_delimiter"))); @@ -202,6 +204,9 @@ StorageFile::StorageFile(const std::string & table_path_, const std::string & us StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArguments args) : StorageFile(args) { + fmt::print(stderr, "Create storage File '{}' from database at \n{}\n", + args.table_id.getNameForLogs(), StackTrace().toString()); + if (relative_table_dir_path.empty()) throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME); if (args.format_name == "Distributed") @@ -215,6 +220,7 @@ StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArgu StorageFile::StorageFile(CommonArguments args) : IStorage(args.table_id) , format_name(args.format_name) + , format_settings(args.format_settings) , compression_method(args.compression_method) , base_path(args.context.getPath()) { @@ -256,12 +262,12 @@ public: } StorageFileSource( - std::shared_ptr storage_, - const StorageMetadataPtr & metadata_snapshot_, - const Context & context_, - UInt64 max_block_size_, - FilesInfoPtr files_info_, - ColumnsDescription columns_description_) + std::shared_ptr storage_, + const StorageMetadataPtr & metadata_snapshot_, + const Context & context_, + UInt64 max_block_size_, + FilesInfoPtr files_info_, + ColumnsDescription columns_description_) : SourceWithProgress(getHeader(metadata_snapshot_, files_info_->need_path_column, files_info_->need_file_column)) , storage(std::move(storage_)) , metadata_snapshot(metadata_snapshot_) @@ -337,9 +343,11 @@ public: method = chooseCompressionMethod(current_path, storage->compression_method); } - read_buf = wrapReadBufferWithCompressionMethod(std::move(nested_buffer), method); - reader = FormatFactory::instance().getInput( - storage->format_name, *read_buf, metadata_snapshot->getSampleBlock(), context, max_block_size); + read_buf = wrapReadBufferWithCompressionMethod( + std::move(nested_buffer), method); + reader = FormatFactory::instance().getInput(storage->format_name, + *read_buf, metadata_snapshot->getSampleBlock(), context, + max_block_size, storage->format_settings); if (columns_description.hasDefaults()) reader = std::make_shared(reader, columns_description, context); @@ -443,8 +451,11 @@ Pipe StorageFile::read( pipes.reserve(num_streams); for (size_t i = 0; i < num_streams; ++i) + { pipes.emplace_back(std::make_shared( - this_ptr, metadata_snapshot, context, max_block_size, files_info, metadata_snapshot->getColumns())); + this_ptr, metadata_snapshot, context, max_block_size, files_info, + metadata_snapshot->getColumns())); + } return Pipe::unitePipes(std::move(pipes)); } @@ -457,7 +468,8 @@ public: StorageFile & storage_, const StorageMetadataPtr & metadata_snapshot_, const CompressionMethod compression_method, - const Context & context) + const Context & context, + const FormatSettings & format_settings) : storage(storage_) , metadata_snapshot(metadata_snapshot_) , lock(storage.rwlock) @@ -485,7 +497,9 @@ public: write_buf = wrapWriteBufferWithCompressionMethod(std::move(naked_buffer), compression_method, 3); - writer = FormatFactory::instance().getOutput(storage.format_name, *write_buf, metadata_snapshot->getSampleBlock(), context); + writer = FormatFactory::instance().getOutput(storage.format_name, + *write_buf, metadata_snapshot->getSampleBlock(), context, + {}, format_settings); } Block getHeader() const override { return metadata_snapshot->getSampleBlock(); } @@ -534,7 +548,8 @@ BlockOutputStreamPtr StorageFile::write( path = paths[0]; return std::make_shared(*this, metadata_snapshot, - chooseCompressionMethod(path, compression_method), context); + chooseCompressionMethod(path, compression_method), context, + format_settings); } Strings StorageFile::getDataPaths() const @@ -620,6 +635,20 @@ void registerStorageFile(StorageFactory & factory) engine_args_ast[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args_ast[0], factory_args.local_context); storage_args.format_name = engine_args_ast[0]->as().value.safeGet(); + if (factory_args.storage_def->settings) + { + Context local_context_copy = factory_args.local_context; + local_context_copy.applySettingsChanges( + factory_args.storage_def->settings->changes); + storage_args.format_settings = getFormatSettings( + local_context_copy); + } + else + { + storage_args.format_settings = getFormatSettings( + factory_args.local_context); + } + if (engine_args_ast.size() == 1) /// Table in database return StorageFile::create(factory_args.relative_data_path, storage_args); diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index cc8f6dc2b8a..695cd0d3912 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -52,6 +52,7 @@ public: { StorageID table_id; std::string format_name; + FormatSettings format_settings; std::string compression_method; const ColumnsDescription & columns; const ConstraintsDescription & constraints; @@ -79,6 +80,7 @@ private: explicit StorageFile(CommonArguments args); std::string format_name; + FormatSettings format_settings; int table_fd = -1; String compression_method; diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 39de6dce92c..5dbb338b91f 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -1,10 +1,12 @@ -#include -#include -#include -#include #include -#include + #include "registerTableFunctions.h" +#include +#include +#include +#include +#include +#include namespace DB { @@ -12,7 +14,16 @@ StoragePtr TableFunctionFile::getStorage( const String & source, const String & format_, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const std::string & compression_method_) const { - StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), format_, compression_method_, columns, ConstraintsDescription{}, global_context}; + StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), + format_, + getFormatSettings(global_context), + compression_method_, + columns, + ConstraintsDescription{}, + global_context}; + + fmt::print(stderr, "format settings delimiter = '{}'\n", + args.format_settings.csv.delimiter); return StorageFile::create(source, global_context.getUserFilesPath(), args); } From 417fa65cb487e5115281b3b12ba877f2de08cbd0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 30 Oct 2020 16:34:59 +0300 Subject: [PATCH 013/258] fixup --- src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp index 79027b4afac..1c6ed0e843f 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp @@ -90,7 +90,7 @@ void JSONEachRowRowOutputFormat::writePrefix() void JSONEachRowRowOutputFormat::writeSuffix() { - writeCString("\n"); + writeCString("\n", out); if (settings.json.array_of_rows) { writeCString("]\n", out); From 6fb7e944dbf0194cfd526a90e43d53e9acd683d8 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 30 Oct 2020 21:10:02 +0300 Subject: [PATCH 014/258] a test for file engine settings --- src/Storages/StorageFile.cpp | 28 +++++++++---------- .../01544_file_engine_settings.reference | 2 ++ .../0_stateless/01544_file_engine_settings.sh | 23 +++++++++++++++ 3 files changed, 39 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/01544_file_engine_settings.reference create mode 100755 tests/queries/0_stateless/01544_file_engine_settings.sh diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index d175736e8f8..517eeb1e671 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -167,18 +167,18 @@ StorageFile::StorageFile(int table_fd_, CommonArguments args) StorageFile::StorageFile(const std::string & table_path_, const std::string & user_files_path, CommonArguments args) : StorageFile(args) { - fmt::print(stderr, "Create storage File '{}' from file at \n{}\n", - args.table_id.getNameForLogs(), StackTrace().toString()); - - const auto & changes = args.context.getSettings().changes(); - for (const auto & change : changes) - { - fmt::print(stderr, "Changed setting '{}' to '{}'\n", - change.name, toString(change.value)); - } - - fmt::print(stderr, "delimiter = '{}'\n", - toString(args.context.getSettings().get("format_csv_delimiter"))); +// fmt::print(stderr, "Create storage File '{}' from file at \n{}\n", +// args.table_id.getNameForLogs(), StackTrace().toString()); +// +// const auto & changes = args.context.getSettings().changes(); +// for (const auto & change : changes) +// { +// fmt::print(stderr, "Changed setting '{}' to '{}'\n", +// change.name, toString(change.value)); +// } +// +// fmt::print(stderr, "delimiter = '{}'\n", +// toString(args.context.getSettings().get("format_csv_delimiter"))); is_db_table = false; @@ -204,8 +204,8 @@ StorageFile::StorageFile(const std::string & table_path_, const std::string & us StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArguments args) : StorageFile(args) { - fmt::print(stderr, "Create storage File '{}' from database at \n{}\n", - args.table_id.getNameForLogs(), StackTrace().toString()); +// fmt::print(stderr, "Create storage File '{}' from database at \n{}\n", +// args.table_id.getNameForLogs(), StackTrace().toString()); if (relative_table_dir_path.empty()) throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME); diff --git a/tests/queries/0_stateless/01544_file_engine_settings.reference b/tests/queries/0_stateless/01544_file_engine_settings.reference new file mode 100644 index 00000000000..d2afb8fc688 --- /dev/null +++ b/tests/queries/0_stateless/01544_file_engine_settings.reference @@ -0,0 +1,2 @@ +1|1 +1 1 diff --git a/tests/queries/0_stateless/01544_file_engine_settings.sh b/tests/queries/0_stateless/01544_file_engine_settings.sh new file mode 100755 index 00000000000..b13ec0f3db3 --- /dev/null +++ b/tests/queries/0_stateless/01544_file_engine_settings.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eu + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +the_file="$CLICKHOUSE_TMP/01544-t.csv" +rm -f -- "$the_file" + +# We are going to check that format settings work for File engine, +# by creating a table with a non-default delimiter, and reading from it. +${CLICKHOUSE_LOCAL} --query " + create table t(a int, b int) engine File(CSV, '$the_file') settings format_csv_delimiter = '|'; + insert into t select 1 a, 1 b; +" + +# See what's in the file +cat "$the_file" + +${CLICKHOUSE_LOCAL} --query " + create table t(a int, b int) engine File(CSV, '$the_file') settings format_csv_delimiter = '|'; + select * from t; +" From 382fff9009eb001487f1f4f88b390d52717e46a8 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 1 Nov 2020 13:54:07 +0000 Subject: [PATCH 015/258] fix --- .../IInterpreterUnionOrSelectQuery.h | 39 ++ src/Interpreters/InterpreterSelectQuery.cpp | 11 +- src/Interpreters/InterpreterSelectQuery.h | 29 +- .../InterpreterSelectWithUnionQuery.cpp | 413 ++++++++---------- .../InterpreterSelectWithUnionQuery.h | 53 +-- src/Interpreters/executeQuery.cpp | 2 - src/Parsers/ASTSelectWithUnionQuery.cpp | 11 +- src/Parsers/ASTSelectWithUnionQuery.h | 3 - src/Parsers/IAST.cpp | 5 - src/Parsers/IAST.h | 2 - src/Parsers/ParserSelectWithUnionQuery.cpp | 35 +- 11 files changed, 238 insertions(+), 365 deletions(-) create mode 100644 src/Interpreters/IInterpreterUnionOrSelectQuery.h diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.h b/src/Interpreters/IInterpreterUnionOrSelectQuery.h new file mode 100644 index 00000000000..67e3cf25e5a --- /dev/null +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ +class IInterpreterUnionOrSelectQuery : public IInterpreter +{ +public: + IInterpreterUnionOrSelectQuery(const ASTPtr & query_ptr_, const Context & context_, const SelectQueryOptions & options_) + : query_ptr(query_ptr_) + , context(std::make_shared(context_)) + , options(options_) + , max_streams(context->getSettingsRef().max_threads) + { + } + + virtual void buildQueryPlan(QueryPlan & query_plan) = 0; + + virtual void ignoreWithTotals() = 0; + + virtual ~IInterpreterUnionOrSelectQuery() override = default; + + Block getSampleBlock() { return result_header; } + + size_t getMaxStreams() const { return max_streams; } + +protected: + ASTPtr query_ptr; + std::shared_ptr context; + Block result_header; + SelectQueryOptions options; + size_t max_streams = 1; +}; +} + diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index d9821be4e4e..0b86914b1fa 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -119,7 +119,6 @@ String InterpreterSelectQuery::generateFilterActions( ParserExpression expr_parser; expr_list->children.push_back(parseQuery(expr_parser, column_str, 0, context->getSettingsRef().max_parser_depth)); } - select_ast->setExpression(ASTSelectQuery::Expression::TABLES, std::make_shared()); auto tables = select_ast->tables(); auto tables_elem = std::make_shared(); @@ -215,10 +214,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( const SelectQueryOptions & options_, const Names & required_result_column_names, const StorageMetadataPtr & metadata_snapshot_) - : options(options_) /// NOTE: the query almost always should be cloned because it will be modified during analysis. - , query_ptr(options.modify_inplace ? query_ptr_ : query_ptr_->clone()) - , context(std::make_shared(context_)) + : IInterpreterUnionOrSelectQuery(options_.modify_inplace ? query_ptr_ : query_ptr_->clone(), context_, options_) , storage(storage_) , input(input_) , input_pipe(std::move(input_pipe_)) @@ -464,12 +461,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( sanitizeBlock(result_header, true); } - -Block InterpreterSelectQuery::getSampleBlock() -{ - return result_header; -} - void InterpreterSelectQuery::buildQueryPlan(QueryPlan & query_plan) { executeImpl(query_plan, input, std::move(input_pipe)); diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 455b1a1e623..6bbcb9a6bcd 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -3,16 +3,15 @@ #include #include -#include #include #include #include -#include -#include +#include +#include +#include +#include #include #include -#include -#include #include @@ -32,7 +31,7 @@ using TreeRewriterResultPtr = std::shared_ptr; /** Interprets the SELECT query. Returns the stream of blocks with the results of the query before `to_stage` stage. */ -class InterpreterSelectQuery : public IInterpreter +class InterpreterSelectQuery : public IInterpreterUnionOrSelectQuery { public: /** @@ -79,18 +78,12 @@ public: BlockIO execute() override; /// Builds QueryPlan for current query. - void buildQueryPlan(QueryPlan & query_plan); + virtual void buildQueryPlan(QueryPlan & query_plan) override; bool ignoreLimits() const override { return options.ignore_limits; } bool ignoreQuota() const override { return options.ignore_quota; } - Block getSampleBlock(); - - void ignoreWithTotals(); - - ASTPtr getQuery() const { return query_ptr; } - - size_t getMaxStreams() const { return max_streams; } + virtual void ignoreWithTotals() override; const SelectQueryInfo & getQueryInfo() const { return query_info; } @@ -158,9 +151,6 @@ private: */ void initSettings(); - SelectQueryOptions options; - ASTPtr query_ptr; - std::shared_ptr context; TreeRewriterResultPtr syntax_analyzer_result; std::unique_ptr query_analyzer; SelectQueryInfo query_info; @@ -172,15 +162,10 @@ private: QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; - /// How many streams we ask for storage to produce, and in how many threads we will do further processing. - size_t max_streams = 1; - /// List of columns to read to execute the query. Names required_columns; /// Structure of query source (table, subquery, etc). Block source_header; - /// Structure of query result. - Block result_header; /// The subquery interpreter, if the subquery std::unique_ptr interpreter_subquery; diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 92f88342241..43ff1c48167 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -10,8 +10,6 @@ #include #include -#include - namespace DB { @@ -22,206 +20,40 @@ namespace ErrorCodes extern const int EXPECTED_ALL_OR_DISTINCT; } -struct CustomizeUnionModeRewrite -{ - using TypeToVisit = ASTSelectWithUnionQuery; - - const UnionMode & default_union_mode; - - void visit(ASTSelectWithUnionQuery & union_select, ASTPtr &) - { - size_t num_selects = union_select.list_of_selects->children.size(); - if (!num_selects) - throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); - if (num_selects > 1) - { - for (auto & mode : union_select.union_modes) - { - if (mode == ASTSelectWithUnionQuery::Mode::Unspecified) - { - if (default_union_mode == UnionMode::ALL) - mode = ASTSelectWithUnionQuery::Mode::ALL; - else if (default_union_mode == UnionMode::DISTINCT) - mode = ASTSelectWithUnionQuery::Mode::DISTINCT; - else - throw Exception( - "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", - DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); - } - } - /// Optimize: if there is UNION DISTINCT, all previous UNION DISTINCT can be rewritten to UNION ALL. - /// Therefore we have at most one UNION DISTINCT in a sequence. - for (auto rit = union_select.union_modes.rbegin(); rit != union_select.union_modes.rend(); ++rit) - { - if (*rit == ASTSelectWithUnionQuery::Mode::DISTINCT) - { - /// Number of streams need to do a DISTINCT transform after unite - for (auto mode_to_modify = ++rit; mode_to_modify != union_select.union_modes.rend(); ++mode_to_modify) - *mode_to_modify = ASTSelectWithUnionQuery::Mode::ALL; - break; - } - } - } - } -}; - -using CustomizeUnionQueryOptimizeVisitor = InDepthNodeVisitor, true>; - -QueryPlan NestedInterpreter::buildQueryPlan(const std::shared_ptr & context, const Block & header) -{ - QueryPlan res; - if (type == Type::LEAF) - { - if (interpreter) - { - interpreter->buildQueryPlan(res); - return res; - } - else - throw Exception("Interpreter is not initialized.", ErrorCodes::LOGICAL_ERROR); - } - - if (num_distinct_union == 0) - { - std::vector> plans(children.size()); - DataStreams data_streams(children.size()); - - for (size_t i = 0; i < children.size(); ++i) - { - plans[i] = std::make_unique(children[i]->buildQueryPlan(context, header)); - data_streams[i] = plans[i]->getCurrentDataStream(); - } - - auto max_threads = context->getSettingsRef().max_threads; - auto union_step = std::make_unique(std::move(data_streams), header, max_threads); - - res.unitePlans(std::move(union_step), std::move(plans)); - return res; - } - /// The first union_distinct_num UNION streams need to do a DISTINCT transform after unite - else - { - QueryPlan distinct_query_plan; - - std::vector> plans(num_distinct_union); - DataStreams data_streams(num_distinct_union); - - for (size_t i = 0; i < num_distinct_union; ++i) - { - plans[i] = std::make_unique(children[i]->buildQueryPlan(context, header)); - data_streams[i] = plans[i]->getCurrentDataStream(); - } - - auto max_threads = context->getSettingsRef().max_threads; - auto union_step = std::make_unique(std::move(data_streams), header, max_threads); - - distinct_query_plan.unitePlans(std::move(union_step), std::move(plans)); - - /// Add distinct transform - const Settings & settings = context->getSettingsRef(); - SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); - - auto distinct_step - = std::make_unique(distinct_query_plan.getCurrentDataStream(), limits, 0, header.getNames(), false); - - distinct_query_plan.addStep(std::move(distinct_step)); - - /// No other UNION streams after DISTINCT stream - if (num_distinct_union == children.size()) - { - return distinct_query_plan; - } - - /// Build final UNION step - std::vector> final_plans(children.size() - num_distinct_union + 1); - DataStreams final_data_streams(children.size() - num_distinct_union + 1); - - final_plans[0] = std::make_unique(std::move(distinct_query_plan)); - final_data_streams[0] = final_plans[0]->getCurrentDataStream(); - - for (size_t i = 1; i < children.size() - num_distinct_union + 1; ++i) - { - final_plans[i] = std::make_unique(children[num_distinct_union + i - 1]->buildQueryPlan(context, header)); - final_data_streams[i] = final_plans[i]->getCurrentDataStream(); - } - - auto final_union_step = std::make_unique(std::move(final_data_streams), header, max_threads); - res.unitePlans(std::move(final_union_step), std::move(final_plans)); - return res; - } -} - -void NestedInterpreter::ignoreWithTotals() -{ - if (type == Type::LEAF) - { - if (interpreter) - interpreter->ignoreWithTotals(); - else - { - throw Exception("Interpreter is not initialized.", ErrorCodes::LOGICAL_ERROR); - } - return; - } - for (auto & child : children) - { - child->ignoreWithTotals(); - } -} - - InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( - const ASTPtr & query_ptr_, - const Context & context_, - const SelectQueryOptions & options_, - const Names & required_result_column_names) - : options(options_), - query_ptr(query_ptr_), - context(std::make_shared(context_)), - max_streams(context->getSettingsRef().max_threads) + const ASTPtr & query_ptr_, const Context & context_, const SelectQueryOptions & options_, const Names & required_result_column_names) + : IInterpreterUnionOrSelectQuery(query_ptr_, context_, options_) { - std::cout << "\n\n In InterpreterSelectWithUnionQuery\n\n"; const auto & ast = query_ptr->as(); - std::cout << "\n\n before throw\n\n"; - if (!ast.flatten_nodes_list) - std::cout << "\n\n flatten_nodes_list is null\n\n"; - size_t total_num_selects = ast.flatten_nodes_list->children.size(); - std::cout << "\n\n after get num throw\n\n"; - if (!total_num_selects) - throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); - std::cout << "\n\n after throw\n\n"; - /// Rewrite ast with settings.union_default_mode - const auto & settings = context->getSettingsRef(); - CustomizeUnionQueryOptimizeVisitor::Data data_union_mode{settings.union_default_mode}; - CustomizeUnionQueryOptimizeVisitor(data_union_mode).visit(query_ptr); + size_t num_children = ast.list_of_selects->children.size(); + if (!num_children) + throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); /// We first build nested interpreters for each select query, then using this nested interpreters to build Tree Structured nested interpreter. /// Note that we pass 'required_result_column_names' to first SELECT. /// And for the rest, we pass names at the corresponding positions of 'required_result_column_names' in the result of first SELECT, /// because names could be different. - std::vector> interpreters; - interpreters.reserve(total_num_selects); - std::vector required_result_column_names_for_other_selects(total_num_selects); - if (!required_result_column_names.empty() && total_num_selects > 1) + nested_interpreters.reserve(num_children); + std::vector required_result_column_names_for_other_selects(num_children); + + if (!required_result_column_names.empty() && num_children > 1) { /// Result header if there are no filtering by 'required_result_column_names'. /// We use it to determine positions of 'required_result_column_names' in SELECT clause. - Block full_result_header - = InterpreterSelectQuery(ast.flatten_nodes_list->children.at(0), *context, options.copy().analyze().noModify()) - .getSampleBlock(); + Block full_result_header = getCurrentChildResultHeader(ast.list_of_selects->children.at(0), required_result_column_names); std::vector positions_of_required_result_columns(required_result_column_names.size()); + for (size_t required_result_num = 0, size = required_result_column_names.size(); required_result_num < size; ++required_result_num) positions_of_required_result_columns[required_result_num] = full_result_header.getPositionByName(required_result_column_names[required_result_num]); - for (size_t query_num = 1; query_num < total_num_selects; ++query_num) + for (size_t query_num = 1; query_num < num_children; ++query_num) { Block full_result_header_for_current_select - = InterpreterSelectQuery(ast.flatten_nodes_list->children.at(query_num), *context, options.copy().analyze().noModify()) - .getSampleBlock(); + = getCurrentChildResultHeader(ast.list_of_selects->children.at(query_num), required_result_column_names); if (full_result_header_for_current_select.columns() != full_result_header.columns()) throw Exception("Different number of columns in UNION ALL elements:\n" @@ -236,26 +68,26 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( } } - for (size_t query_num = 0; query_num < total_num_selects; ++query_num) + for (size_t query_num = 0; query_num < num_children; ++query_num) { const Names & current_required_result_column_names = query_num == 0 ? required_result_column_names : required_result_column_names_for_other_selects[query_num]; - interpreters.emplace_back(std::make_shared( - ast.flatten_nodes_list->children.at(query_num), *context, options, current_required_result_column_names)); + nested_interpreters.emplace_back( + buildCurrentChildInterpreter(ast.list_of_selects->children.at(query_num), current_required_result_column_names)); } /// Determine structure of the result. - if (total_num_selects == 1) + if (num_children == 1) { - result_header = interpreters.front()->getSampleBlock(); + result_header = nested_interpreters.front()->getSampleBlock(); } else { - Blocks headers(total_num_selects); - for (size_t query_num = 0; query_num < total_num_selects; ++query_num) - headers[query_num] = interpreters[query_num]->getSampleBlock(); + Blocks headers(num_children); + for (size_t query_num = 0; query_num < num_children; ++query_num) + headers[query_num] = nested_interpreters[query_num]->getSampleBlock(); result_header = getCommonHeaderForUnion(headers); } @@ -263,7 +95,7 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( /// InterpreterSelectWithUnionQuery ignores limits if all nested interpreters ignore limits. bool all_nested_ignore_limits = true; bool all_nested_ignore_quota = true; - for (auto & interpreter : interpreters) + for (auto & interpreter : nested_interpreters) { if (!interpreter->ignoreLimits()) all_nested_ignore_limits = false; @@ -273,44 +105,6 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( options.ignore_limits |= all_nested_ignore_limits; options.ignore_quota |= all_nested_ignore_quota; - int index = 0; - buildNestedTreeInterpreter(query_ptr, nested_interpreter, interpreters, index); -} - -/// We build a Tree Structured nested interpreters to build QueryPlan later -/// The structure of build nested interpreters is same as AST Tree -void InterpreterSelectWithUnionQuery::buildNestedTreeInterpreter( - const ASTPtr & ast_ptr, - std::shared_ptr nested_interpreter_, - std::vector> & interpreters, - int & index) -{ - std::cout << "\n\n in build \n\n"; - if (auto inner_union = ast_ptr->as()) - { - auto internal_intepreter = std::make_shared(); - const auto & union_modes = inner_union->union_modes; - - for (auto rit = union_modes.rbegin(); rit != union_modes.rend(); ++rit) - { - if (*rit == ASTSelectWithUnionQuery::Mode::DISTINCT) - { - internal_intepreter->num_distinct_union = union_modes.rend() - rit + 1; - break; - } - } - - nested_interpreter_->children.push_back(internal_intepreter); - - for (auto & child : inner_union->list_of_selects->children) - buildNestedTreeInterpreter(child, internal_intepreter, interpreters, index); - return; - } - - auto leaf_interpreter = std::make_shared(); - leaf_interpreter->type = NestedInterpreter::Type::LEAF; - leaf_interpreter->interpreter = interpreters[index++]; - nested_interpreter_->children.push_back(leaf_interpreter); } Block InterpreterSelectWithUnionQuery::getCommonHeaderForUnion(const Blocks & headers) @@ -343,33 +137,167 @@ Block InterpreterSelectWithUnionQuery::getCommonHeaderForUnion(const Blocks & he return common_header; } +Block InterpreterSelectWithUnionQuery::getCurrentChildResultHeader(const ASTPtr & ast_ptr_, const Names & required_result_column_names) +{ + if (const auto _ = ast_ptr_->as()) + return InterpreterSelectWithUnionQuery(ast_ptr_, *context, options.copy().analyze().noModify(), required_result_column_names) + .getSampleBlock(); + else + return InterpreterSelectQuery(ast_ptr_, *context, options.copy().analyze().noModify()).getSampleBlock(); +} + +std::unique_ptr +InterpreterSelectWithUnionQuery::buildCurrentChildInterpreter(const ASTPtr & ast_ptr_, const Names & current_required_result_column_names) +{ + if (const auto _ = ast_ptr_->as()) + return std::make_unique(ast_ptr_, *context, options, current_required_result_column_names); + else + return std::make_unique(ast_ptr_, *context, options, current_required_result_column_names); +} InterpreterSelectWithUnionQuery::~InterpreterSelectWithUnionQuery() = default; - -Block InterpreterSelectWithUnionQuery::getSampleBlock() +Block InterpreterSelectWithUnionQuery::getSampleBlock(const ASTPtr & query_ptr_, const Context & context_) { - return result_header; -} - -Block InterpreterSelectWithUnionQuery::getSampleBlock( - const ASTPtr & query_ptr, - const Context & context) -{ - auto & cache = context.getSampleBlockCache(); + auto & cache = context_.getSampleBlockCache(); /// Using query string because query_ptr changes for every internal SELECT - auto key = queryToString(query_ptr); + auto key = queryToString(query_ptr_); if (cache.find(key) != cache.end()) { return cache[key]; } - return cache[key] = InterpreterSelectWithUnionQuery(query_ptr, context, SelectQueryOptions().analyze()).getSampleBlock(); + return cache[key] = InterpreterSelectWithUnionQuery(query_ptr_, context_, SelectQueryOptions().analyze()).getSampleBlock(); +} + +size_t InterpreterSelectWithUnionQuery::optimizeUnionList() +{ + auto union_distinct_num = 0; + + auto union_default_mode = context->getSettingsRef().union_default_mode; + auto & ast = query_ptr->as(); + size_t num_selects = ast.list_of_selects->children.size(); + + if (!num_selects) + throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); + + if (num_selects > 1) + { + for (auto & mode : ast.union_modes) + { + if (mode == ASTSelectWithUnionQuery::Mode::Unspecified) + { + if (union_default_mode == UnionMode::ALL) + mode = ASTSelectWithUnionQuery::Mode::ALL; + else if (union_default_mode == UnionMode::DISTINCT) + mode = ASTSelectWithUnionQuery::Mode::DISTINCT; + else + throw Exception( + "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", + DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); + } + } + /// Optimize: if there is UNION DISTINCT, all previous UNION DISTINCT can be rewritten to UNION ALL. + /// Therefore we have at most one UNION DISTINCT in a sequence. + for (auto rit = ast.union_modes.rbegin(); rit != ast.union_modes.rend(); ++rit) + { + if (*rit == ASTSelectWithUnionQuery::Mode::DISTINCT) + { + /// Number of streams need to do a DISTINCT transform after unite + union_distinct_num = ast.union_modes.rend() - rit + 1; + for (auto mode_to_modify = ++rit; mode_to_modify != ast.union_modes.rend(); ++mode_to_modify) + *mode_to_modify = ASTSelectWithUnionQuery::Mode::ALL; + break; + } + } + } + return union_distinct_num; } void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) { - query_plan = nested_interpreter->buildQueryPlan(context, result_header); + auto num_distinct_union = optimizeUnionList(); + size_t num_plans = nested_interpreters.size(); + + /// Skip union for single interpreter. + if (num_plans == 1) + { + nested_interpreters.front()->buildQueryPlan(query_plan); + return; + } + + /// All UNION streams in the chain does not need to do DISTINCT transform + if (num_distinct_union == 0) + { + std::vector> plans(num_plans); + DataStreams data_streams(num_plans); + + for (size_t i = 0; i < num_plans; ++i) + { + plans[i] = std::make_unique(); + nested_interpreters[i]->buildQueryPlan(*plans[i]); + data_streams[i] = plans[i]->getCurrentDataStream(); + } + + auto max_threads = context->getSettingsRef().max_threads; + auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); + + query_plan.unitePlans(std::move(union_step), std::move(plans)); + } + + /// The first union_distinct_num UNION streams need to do a DISTINCT transform after unite + else + { + QueryPlan distinct_query_plan; + + std::vector> plans(num_distinct_union); + DataStreams data_streams(num_distinct_union); + + for (size_t i = 0; i < num_distinct_union; ++i) + { + plans[i] = std::make_unique(); + nested_interpreters[i]->buildQueryPlan(*plans[i]); + data_streams[i] = plans[i]->getCurrentDataStream(); + } + + auto max_threads = context->getSettingsRef().max_threads; + auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); + + distinct_query_plan.unitePlans(std::move(union_step), std::move(plans)); + + /// Add distinct transform + const Settings & settings = context->getSettingsRef(); + SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); + + auto distinct_step + = std::make_unique(distinct_query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false); + + distinct_query_plan.addStep(std::move(distinct_step)); + + /// No other UNION streams after DISTINCT stream + if (num_plans == num_distinct_union) + { + query_plan = std::move(distinct_query_plan); + return; + } + + /// Build final UNION step + std::vector> final_plans(num_plans - num_distinct_union + 1); + DataStreams final_data_streams(num_plans - num_distinct_union + 1); + + final_plans[0] = std::make_unique(std::move(distinct_query_plan)); + final_data_streams[0] = final_plans[0]->getCurrentDataStream(); + + for (size_t i = 1; i < num_plans - num_distinct_union + 1; ++i) + { + final_plans[i] = std::make_unique(); + nested_interpreters[num_distinct_union + i - 1]->buildQueryPlan(*final_plans[i]); + final_data_streams[i] = final_plans[i]->getCurrentDataStream(); + } + + auto final_union_step = std::make_unique(std::move(final_data_streams), result_header, max_threads); + query_plan.unitePlans(std::move(final_union_step), std::move(final_plans)); + } } BlockIO InterpreterSelectWithUnionQuery::execute() @@ -390,7 +318,8 @@ BlockIO InterpreterSelectWithUnionQuery::execute() void InterpreterSelectWithUnionQuery::ignoreWithTotals() { - nested_interpreter->ignoreWithTotals(); + for (auto & interpreter : nested_interpreters) + interpreter->ignoreWithTotals(); } } diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h index 4af73b3c723..3976b5f8b82 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -1,9 +1,7 @@ #pragma once #include -#include -#include -#include +#include #include namespace DB @@ -13,27 +11,13 @@ class Context; class InterpreterSelectQuery; class QueryPlan; -struct NestedInterpreter -{ - ~NestedInterpreter() { } - enum class Type - { - LEAF, - INTERNAL - }; - Type type = Type::INTERNAL; - std::vector> children; - std::shared_ptr interpreter; - size_t num_distinct_union = 0; - QueryPlan buildQueryPlan(const std::shared_ptr & context, const Block & header); - void ignoreWithTotals(); -}; - /** Interprets one or multiple SELECT queries inside UNION/UNION ALL/UNION DISTINCT chain. */ -class InterpreterSelectWithUnionQuery : public IInterpreter +class InterpreterSelectWithUnionQuery : public IInterpreterUnionOrSelectQuery { public: + using IInterpreterUnionOrSelectQuery::getSampleBlock; + InterpreterSelectWithUnionQuery( const ASTPtr & query_ptr_, const Context & context_, @@ -43,41 +27,30 @@ public: ~InterpreterSelectWithUnionQuery() override; /// Builds QueryPlan for current query. - void buildQueryPlan(QueryPlan & query_plan); + virtual void buildQueryPlan(QueryPlan & query_plan) override; BlockIO execute() override; bool ignoreLimits() const override { return options.ignore_limits; } bool ignoreQuota() const override { return options.ignore_quota; } - Block getSampleBlock(); - static Block getSampleBlock( const ASTPtr & query_ptr_, const Context & context_); - void ignoreWithTotals(); - - ASTPtr getQuery() const { return query_ptr; } + virtual void ignoreWithTotals() override; private: - SelectQueryOptions options; - ASTPtr query_ptr; - std::shared_ptr context; - - std::shared_ptr nested_interpreter; - - Block result_header; - - size_t max_streams = 1; + std::vector> nested_interpreters; static Block getCommonHeaderForUnion(const Blocks & headers); - static void buildNestedTreeInterpreter( - const ASTPtr & ast_ptr, - std::shared_ptr nested_interpreter_, - std::vector> & interpreters, - int & index); + Block getCurrentChildResultHeader(const ASTPtr & ast_ptr_, const Names & required_result_column_names); + + std::unique_ptr + buildCurrentChildInterpreter(const ASTPtr & ast_ptr_, const Names & current_required_result_column_names); + + size_t optimizeUnionList(); }; } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 8b4b35785c1..57c557c5658 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -70,12 +70,10 @@ namespace ErrorCodes static void checkASTSizeLimits(const IAST & ast, const Settings & settings) { - std::cout << "\n\n before check limits"; if (settings.max_ast_depth) ast.checkDepth(settings.max_ast_depth); if (settings.max_ast_elements) ast.checkSize(settings.max_ast_elements); - std::cout << "\n\n after check limits"; } diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 5deae6f653f..8748bf1ef85 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -15,8 +15,7 @@ ASTPtr ASTSelectWithUnionQuery::clone() const res->list_of_selects = list_of_selects->clone(); res->children.push_back(res->list_of_selects); - res->union_modes.insert(res->union_modes.begin(), union_modes.begin(), union_modes.end()); - res->flatten_nodes_list = flatten_nodes_list->clone(); + res->union_modes = union_modes; cloneOutputOptions(*res); return res; @@ -25,10 +24,8 @@ ASTPtr ASTSelectWithUnionQuery::clone() const void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - std::cout << "\n\nin format \n\n"; std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); -#if 0 auto mode_to_str = [&](auto mode) { if (mode == Mode::Unspecified) @@ -38,18 +35,16 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F else return "DISTINCT"; }; -#endif - for (ASTs::const_iterator it = flatten_nodes_list->children.begin(); it != flatten_nodes_list->children.end(); ++it) + for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) { if (it != list_of_selects->children.begin()) settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " - // << mode_to_str(union_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : "") + << mode_to_str(union_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : "") << settings.nl_or_ws; (*it)->formatImpl(settings, state, frame); } - std::cout << "\n\nafter format \n\n"; } } diff --git a/src/Parsers/ASTSelectWithUnionQuery.h b/src/Parsers/ASTSelectWithUnionQuery.h index 67ec21e246c..5600dd4b43a 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.h +++ b/src/Parsers/ASTSelectWithUnionQuery.h @@ -28,9 +28,6 @@ public: Modes union_modes; ASTPtr list_of_selects; - - /// we need flatten_nodes to help build nested_interpreter - ASTPtr flatten_nodes_list; }; } diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index d9f0b3562bc..8ee4154541b 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -76,18 +76,13 @@ void IAST::updateTreeHashImpl(SipHash & hash_state) const size_t IAST::checkDepthImpl(size_t max_depth, size_t level) const { - std::cout << "\n\n in check depth impl\n\n"; - std::cout << "\nchildren.size = " << children.size() << "\n\n"; size_t res = level + 1; for (const auto & child : children) { - std::cout << "\n in for\n\n"; if (level >= max_depth) throw Exception("AST is too deep. Maximum: " + toString(max_depth), ErrorCodes::TOO_DEEP_AST); res = std::max(res, child->checkDepthImpl(max_depth, level + 1)); - std::cout << "\n after for\n\n"; } - std::cout << "\n\n after impl\n\n"; return res; } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index d1fca853592..c88c80021d6 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -11,7 +11,6 @@ #include #include -#include class SipHash; @@ -92,7 +91,6 @@ public: */ size_t checkDepth(size_t max_depth) const { - std::cout << "\n in check depth\n\n"; return checkDepthImpl(max_depth, 0); } diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index ee03da753e4..382d7a66669 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -1,5 +1,3 @@ -#include -#include #include #include #include @@ -10,24 +8,9 @@ namespace DB { -static void getSelectsFromUnionListNode(ASTPtr & ast_select, ASTs & selects) -{ - if (auto * inner_union = ast_select->as()) - { - for (auto & child : inner_union->list_of_selects->children) - { - getSelectsFromUnionListNode(child, selects); - } - - return; - } - - selects.push_back(std::move(ast_select)); -} bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - std::cout << "\n\n in ParserSelectWithUnionQuery\n\n"; ASTPtr list_node; ParserUnionList parser( @@ -42,27 +25,17 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & auto select_with_union_query = std::make_shared(); node = select_with_union_query; - select_with_union_query->list_of_selects = std::make_shared(); + select_with_union_query->list_of_selects = list_node; select_with_union_query->children.push_back(select_with_union_query->list_of_selects); - select_with_union_query->list_of_selects->children.insert( - select_with_union_query->list_of_selects->children.begin(), list_node->children.begin(), list_node->children.end()); select_with_union_query->union_modes = parser.getUnionModes(); - /// NOTE: We cann't simply flatten inner union query now, since we may have different union mode in query, + /// NOTE: We cann't flatten inner union query now, since we may have different union mode in query, /// so flatten may change it's semantics. For example: /// flatten `SELECT 1 UNION (SELECT 1 UNION ALL SELETC 1)` -> `SELECT 1 UNION SELECT 1 UNION ALL SELECT 1` - /// We can use a non-flatten AST to help build QueryPlan in InterpreterSelectWithUnionQuery - - select_with_union_query->flatten_nodes_list = std::make_shared(); - - for (auto & child : list_node->children) - { - getSelectsFromUnionListNode(child, select_with_union_query->flatten_nodes_list->children); - } - std::cout << "\n\n after ParserSelectWithUnionQuery\n\n"; - std::cout << "\n\n flatten_nodes.size =" << select_with_union_query->flatten_nodes_list->children.size() << "\n\n"; return true; } + + } From af45cddbc7a5033296f88259d4a2b79179cda30d Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 1 Nov 2020 14:43:29 +0000 Subject: [PATCH 016/258] fix --- src/Parsers/ASTSelectWithUnionQuery.cpp | 21 +- ...t_and_setting_union_default_mode.reference | 199 +++++++----------- ...istinct_and_setting_union_default_mode.sql | 22 ++ 3 files changed, 109 insertions(+), 133 deletions(-) diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 8748bf1ef85..57a3c00543c 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -1,9 +1,8 @@ -#include #include +#include +#include #include -#include - namespace DB { @@ -40,10 +39,18 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F { if (it != list_of_selects->children.begin()) settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " - << mode_to_str(union_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : "") - << settings.nl_or_ws; - - (*it)->formatImpl(settings, state, frame); + << mode_to_str(union_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : ""); + if (auto _ = (*it)->as()) + { + auto sub_query = std::make_shared(); + sub_query->children.push_back(*it); + sub_query->formatImpl(settings, state, frame); + } + else + { + settings.ostr << settings.nl_or_ws; + (*it)->formatImpl(settings, state, frame); + } } } diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference index 6c4547333fe..868cadf1d81 100644 --- a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference @@ -1,140 +1,87 @@ -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 -0 1 -2 -3 -4 -5 -6 -7 -8 -9 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql index 12fe204591c..30ccfdc8c45 100644 --- a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql @@ -1,2 +1,24 @@ SELECT 1; (((((((SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1; +(((((((SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1; +SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; +SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1; +SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; +SELECT 1 UNION (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1 UNION ALL SELECT 1); +SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1 UNION ALL SELECT 1); +SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1))))))); +SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1))))))); +SELECT * FROM (SELECT 1 UNION ALL (SELECT 1 UNION SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1))); + +SET union_default_mode='ALL'; + +(((((((SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1; +(((((((SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1; +SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; +SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1; +SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; +SELECT 1 UNION (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1 UNION ALL SELECT 1); +SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1 UNION ALL SELECT 1); +SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1))))))); +SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL(SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1))))))); +SELECT * FROM (SELECT 1 UNION ALL (SELECT 1 UNION SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1))); From e82ce52e2f6a458a62fae6171920a57ddbec54a7 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 1 Nov 2020 14:57:41 +0000 Subject: [PATCH 017/258] fix --- src/Interpreters/InterpreterSelectQuery.cpp | 1 + .../InterpreterSelectWithUnionQuery.cpp | 14 +++++++------- src/Interpreters/InterpreterSelectWithUnionQuery.h | 1 - src/Parsers/ExpressionListParsers.cpp | 2 -- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 0b86914b1fa..73252894aa5 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -119,6 +119,7 @@ String InterpreterSelectQuery::generateFilterActions( ParserExpression expr_parser; expr_list->children.push_back(parseQuery(expr_parser, column_str, 0, context->getSettingsRef().max_parser_depth)); } + select_ast->setExpression(ASTSelectQuery::Expression::TABLES, std::make_shared()); auto tables = select_ast->tables(); auto tables_elem = std::make_shared(); diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 43ff1c48167..a3798e42f02 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -1,14 +1,15 @@ -#include -#include -#include -#include #include -#include +#include +#include +#include +#include +#include #include +#include #include #include #include -#include +#include namespace DB { @@ -30,7 +31,6 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( if (!num_children) throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); - /// We first build nested interpreters for each select query, then using this nested interpreters to build Tree Structured nested interpreter. /// Note that we pass 'required_result_column_names' to first SELECT. /// And for the rest, we pass names at the corresponding positions of 'required_result_column_names' in the result of first SELECT, /// because names could be different. diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h index 3976b5f8b82..06d31c92a67 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -2,7 +2,6 @@ #include #include -#include namespace DB { diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 220d304751e..1cc72f5fb8b 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -1,5 +1,3 @@ -#include -#include #include #include From 5f25f9c814a2314489e1f66c8fb52da5fff6ade7 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 1 Nov 2020 15:15:51 +0000 Subject: [PATCH 018/258] fix fix --- src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 2 +- src/Parsers/ParserSelectWithUnionQuery.cpp | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index a3798e42f02..1fecd221aba 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -197,7 +197,7 @@ size_t InterpreterSelectWithUnionQuery::optimizeUnionList() DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); } } - /// Optimize: if there is UNION DISTINCT, all previous UNION DISTINCT can be rewritten to UNION ALL. + /// Optimize general cases: if there is UNION DISTINCT, all previous UNION DISTINCT can be rewritten to UNION ALL. /// Therefore we have at most one UNION DISTINCT in a sequence. for (auto rit = ast.union_modes.rbegin(); rit != ast.union_modes.rend(); ++rit) { diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index 382d7a66669..db586867db0 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -3,8 +3,6 @@ #include #include #include -#include - namespace DB { @@ -32,10 +30,7 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & /// NOTE: We cann't flatten inner union query now, since we may have different union mode in query, /// so flatten may change it's semantics. For example: /// flatten `SELECT 1 UNION (SELECT 1 UNION ALL SELETC 1)` -> `SELECT 1 UNION SELECT 1 UNION ALL SELECT 1` - return true; } - - } From f04d74172f794b76c393036ec2014a1d639d8b11 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 1 Nov 2020 15:41:31 +0000 Subject: [PATCH 019/258] fix --- ...9_union_distinct_and_setting_union_default_mode.reference | 5 +++++ .../01529_union_distinct_and_setting_union_default_mode.sql | 2 ++ 2 files changed, 7 insertions(+) diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference index 868cadf1d81..f9f3ee818e9 100644 --- a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference @@ -10,6 +10,8 @@ 1 1 1 +all +all 1 1 1 @@ -51,6 +53,9 @@ 1 1 1 +all +all +all 1 1 1 diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql index 30ccfdc8c45..6e45c150508 100644 --- a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql @@ -2,6 +2,7 @@ SELECT 1; (((((((SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1; (((((((SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1; SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; +SELECT 'all' UNION SELECT 'all' UNION ALL SELECT 'all'; SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1; SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; SELECT 1 UNION (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1 UNION ALL SELECT 1); @@ -15,6 +16,7 @@ SET union_default_mode='ALL'; (((((((SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1; (((((((SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1; SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; +SELECT 'all' UNION SELECT 'all' UNION ALL SELECT 'all'; SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1; SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; SELECT 1 UNION (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1 UNION ALL SELECT 1); From 3dce3c6a21ed0d16c19e401b66b9b9a27a2e5949 Mon Sep 17 00:00:00 2001 From: feng lv Date: Mon, 2 Nov 2020 05:28:37 +0000 Subject: [PATCH 020/258] fix --- src/Interpreters/InterpreterInsertQuery.cpp | 79 +++++++++++---------- src/Parsers/ASTSelectWithUnionQuery.cpp | 3 +- src/Storages/SelectQueryDescription.cpp | 23 ++++-- 3 files changed, 63 insertions(+), 42 deletions(-) diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index aa8bcd74ea6..5ffd667bf40 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -140,34 +140,39 @@ Block InterpreterInsertQuery::getSampleBlock( /** A query that just reads all data without any complex computations or filetering. * If we just pipe the result to INSERT, we don't have to use too many threads for read. */ -static bool isTrivialSelect(const ASTSelectQuery & select_query) +static bool isTrivialSelect(const ASTPtr & select) { - const auto & tables = select_query.tables(); + if (auto select_query = select->as()) + { + const auto & tables = select_query->tables(); - if (!tables) - return false; + if (!tables) + return false; - const auto & tables_in_select_query = tables->as(); + const auto & tables_in_select_query = tables->as(); - if (tables_in_select_query.children.size() != 1) - return false; + if (tables_in_select_query.children.size() != 1) + return false; - const auto & child = tables_in_select_query.children.front(); - const auto & table_element = child->as(); - const auto & table_expr = table_element.table_expression->as(); + const auto & child = tables_in_select_query.children.front(); + const auto & table_element = child->as(); + const auto & table_expr = table_element.table_expression->as(); - if (table_expr.subquery) - return false; + if (table_expr.subquery) + return false; - /// Note: how to write it in more generic way? - return (!select_query.distinct - && !select_query.limit_with_ties - && !select_query.prewhere() - && !select_query.where() - && !select_query.groupBy() - && !select_query.having() - && !select_query.orderBy() - && !select_query.limitBy()); + /// Note: how to write it in more generic way? + return (!select_query->distinct + && !select_query->limit_with_ties + && !select_query->prewhere() + && !select_query->where() + && !select_query->groupBy() + && !select_query->having() + && !select_query->orderBy() + && !select_query->limitBy()); + } + /// This query is ASTSelectWithUnionQuery subquery + return false; }; @@ -196,23 +201,25 @@ BlockIO InterpreterInsertQuery::execute() auto new_query = std::dynamic_pointer_cast(query.clone()); if (select.list_of_selects->children.size() == 1) { - auto & select_query = select.list_of_selects->children.at(0)->as(); - JoinedTables joined_tables(Context(context), select_query); - - if (joined_tables.tablesCount() == 1) + if (auto select_query = select.list_of_selects->children.at(0)->as()) { - storage_src = std::dynamic_pointer_cast(joined_tables.getLeftTableStorage()); - if (storage_src) + JoinedTables joined_tables(Context(context), *select_query); + + if (joined_tables.tablesCount() == 1) { - const auto select_with_union_query = std::make_shared(); - select_with_union_query->list_of_selects = std::make_shared(); + storage_src = std::dynamic_pointer_cast(joined_tables.getLeftTableStorage()); + if (storage_src) + { + const auto select_with_union_query = std::make_shared(); + select_with_union_query->list_of_selects = std::make_shared(); - auto new_select_query = std::dynamic_pointer_cast(select_query.clone()); - select_with_union_query->list_of_selects->children.push_back(new_select_query); + auto new_select_query = std::dynamic_pointer_cast(select_query->clone()); + select_with_union_query->list_of_selects->children.push_back(new_select_query); - new_select_query->replaceDatabaseAndTable(storage_src->getRemoteDatabaseName(), storage_src->getRemoteTableName()); + new_select_query->replaceDatabaseAndTable(storage_src->getRemoteDatabaseName(), storage_src->getRemoteTableName()); - new_query->select = select_with_union_query; + new_query->select = select_with_union_query; + } } } } @@ -277,10 +284,8 @@ BlockIO InterpreterInsertQuery::execute() { const auto & selects = query.select->as().list_of_selects->children; - is_trivial_insert_select = std::all_of(selects.begin(), selects.end(), [](const ASTPtr & select) - { - return isTrivialSelect(select->as()); - }); + is_trivial_insert_select + = std::all_of(selects.begin(), selects.end(), [](const ASTPtr & select) { return isTrivialSelect(select); }); } if (is_trivial_insert_select) diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 57a3c00543c..29aa1b81acf 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -48,7 +48,8 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F } else { - settings.ostr << settings.nl_or_ws; + if (it != list_of_selects->children.begin()) + settings.ostr << settings.nl_or_ws; (*it)->formatImpl(settings, state, frame); } } diff --git a/src/Storages/SelectQueryDescription.cpp b/src/Storages/SelectQueryDescription.cpp index 0935a5be5ca..db886793f7c 100644 --- a/src/Storages/SelectQueryDescription.cpp +++ b/src/Storages/SelectQueryDescription.cpp @@ -98,20 +98,35 @@ void checkAllowedQueries(const ASTSelectQuery & query) } +/// check if only one single select query in SelectWithUnionQuery +static bool isSingleSelect(const ASTPtr & select, ASTPtr & res) +{ + auto new_select = select->as(); + if (new_select.list_of_selects->children.size() != 1) + return false; + auto & new_inner_query = new_select.list_of_selects->children.at(0); + if (auto _ = new_inner_query->as()) + { + res = new_inner_query; + return true; + } + else + return isSingleSelect(new_inner_query, res); +} + SelectQueryDescription SelectQueryDescription::getSelectQueryFromASTForMatView(const ASTPtr & select, const Context & context) { - auto & new_select = select->as(); + ASTPtr new_inner_query; - if (new_select.list_of_selects->children.size() != 1) + if (!isSingleSelect(select, new_inner_query)) throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); - auto & new_inner_query = new_select.list_of_selects->children.at(0); auto & select_query = new_inner_query->as(); checkAllowedQueries(select_query); SelectQueryDescription result; result.select_table_id = extractDependentTableFromSelectQuery(select_query, context); - result.select_query = new_select.clone(); + result.select_query = select->as().clone(); result.inner_query = new_inner_query->clone(); return result; From 7159affda8e8b91e3e3da8ebcd7681c0ab305c6e Mon Sep 17 00:00:00 2001 From: feng lv Date: Mon, 2 Nov 2020 06:53:09 +0000 Subject: [PATCH 021/258] fix --- src/Parsers/ASTSelectWithUnionQuery.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 29aa1b81acf..54ae310a23a 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -40,11 +40,22 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F if (it != list_of_selects->children.begin()) settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " << mode_to_str(union_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : ""); - if (auto _ = (*it)->as()) + if (auto node = (*it)->as()) { - auto sub_query = std::make_shared(); - sub_query->children.push_back(*it); - sub_query->formatImpl(settings, state, frame); + // just one child in subquery, () is not need + if (node->list_of_selects->children.size() == 1) + { + if (it != list_of_selects->children.begin()) + settings.ostr << settings.nl_or_ws; + node->list_of_selects->children.at(0)->formatImpl(settings, state, frame); + } + // more than one child in subquery + else + { + auto sub_query = std::make_shared(); + sub_query->children.push_back(*it); + sub_query->formatImpl(settings, state, frame); + } } else { From 1a34abadbc25afe8e277273c64d743263ed45616 Mon Sep 17 00:00:00 2001 From: feng lv Date: Mon, 2 Nov 2020 08:02:35 +0000 Subject: [PATCH 022/258] fix fix fix --- src/Interpreters/InterpreterInsertQuery.cpp | 4 ++-- src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 4 ++-- src/Parsers/ASTSelectWithUnionQuery.cpp | 2 +- src/Storages/SelectQueryDescription.cpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 5ffd667bf40..e2830322024 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -142,7 +142,7 @@ Block InterpreterInsertQuery::getSampleBlock( */ static bool isTrivialSelect(const ASTPtr & select) { - if (auto select_query = select->as()) + if (auto * select_query = select->as()) { const auto & tables = select_query->tables(); @@ -201,7 +201,7 @@ BlockIO InterpreterInsertQuery::execute() auto new_query = std::dynamic_pointer_cast(query.clone()); if (select.list_of_selects->children.size() == 1) { - if (auto select_query = select.list_of_selects->children.at(0)->as()) + if (auto * select_query = select.list_of_selects->children.at(0)->as()) { JoinedTables joined_tables(Context(context), *select_query); diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 1fecd221aba..7cc0e890fc2 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -139,7 +139,7 @@ Block InterpreterSelectWithUnionQuery::getCommonHeaderForUnion(const Blocks & he Block InterpreterSelectWithUnionQuery::getCurrentChildResultHeader(const ASTPtr & ast_ptr_, const Names & required_result_column_names) { - if (const auto _ = ast_ptr_->as()) + if (ast_ptr_->as()) return InterpreterSelectWithUnionQuery(ast_ptr_, *context, options.copy().analyze().noModify(), required_result_column_names) .getSampleBlock(); else @@ -149,7 +149,7 @@ Block InterpreterSelectWithUnionQuery::getCurrentChildResultHeader(const ASTPtr std::unique_ptr InterpreterSelectWithUnionQuery::buildCurrentChildInterpreter(const ASTPtr & ast_ptr_, const Names & current_required_result_column_names) { - if (const auto _ = ast_ptr_->as()) + if (ast_ptr_->as()) return std::make_unique(ast_ptr_, *context, options, current_required_result_column_names); else return std::make_unique(ast_ptr_, *context, options, current_required_result_column_names); diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 54ae310a23a..639c8ec1b6e 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -40,7 +40,7 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F if (it != list_of_selects->children.begin()) settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " << mode_to_str(union_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : ""); - if (auto node = (*it)->as()) + if (auto * node = (*it)->as()) { // just one child in subquery, () is not need if (node->list_of_selects->children.size() == 1) diff --git a/src/Storages/SelectQueryDescription.cpp b/src/Storages/SelectQueryDescription.cpp index db886793f7c..c11e6bd74f8 100644 --- a/src/Storages/SelectQueryDescription.cpp +++ b/src/Storages/SelectQueryDescription.cpp @@ -105,7 +105,7 @@ static bool isSingleSelect(const ASTPtr & select, ASTPtr & res) if (new_select.list_of_selects->children.size() != 1) return false; auto & new_inner_query = new_select.list_of_selects->children.at(0); - if (auto _ = new_inner_query->as()) + if (new_inner_query->as()) { res = new_inner_query; return true; From 2a0d9da5e4bd3a2f95615a0bb23cc47c7efb12c4 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 5 Nov 2020 12:25:42 +0300 Subject: [PATCH 023/258] cleanup --- docs/en/sql-reference/statements/select/union-all.md | 2 +- src/Common/ErrorCodes.cpp | 8 -------- src/Interpreters/executeQuery.cpp | 11 +++++++++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-reference/statements/select/union-all.md b/docs/en/sql-reference/statements/select/union-all.md index 1784da37c9a..f150efbdc80 100644 --- a/docs/en/sql-reference/statements/select/union-all.md +++ b/docs/en/sql-reference/statements/select/union-all.md @@ -29,7 +29,7 @@ Queries that are parts of `UNION ALL` can’t be enclosed in round brackets. [OR The difference between `UNION ALL` and `UNION DISTINCT` is that `UNION DISTINCT` will do a distinct transform for union result, it is equivalent to `SELECT DISTINCT` from a subquery containing `UNION ALL`. # UNION Clause {#union-clause} -By defaul, `UNION` has same react as `UNION DISTINCT`, but you can specify union mode by setting `union_default_mode`, values can be 'ALL', 'DISTINCT' or empty string. However, if you use `UNION` with setting `union_default_mode` to empty string, it will throw an exception. +By default, `UNION` has the same behavior as `UNION DISTINCT`, but you can specify union mode by setting `union_default_mode`, values can be 'ALL', 'DISTINCT' or empty string. However, if you use `UNION` with setting `union_default_mode` to empty string, it will throw an exception. ## Implementation Details {#implementation-details} diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 501f655154b..6f46dd861d0 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -559,15 +559,7 @@ namespace ErrorCodes return error_codes_names.names[error_code]; } -<<<<<<< HEAD - extern const int CONDITIONAL_TREE_PARENT_NOT_FOUND = 2001; - extern const int ILLEGAL_PROJECTION_MANIPULATOR = 2002; - extern const int UNRECOGNIZED_ARGUMENTS = 2003; - extern const int UNKNOWN_UNION = 2004; - extern const int EXPECTED_ALL_OR_DISTINCT = 2005; -======= ErrorCode end() { return END+1; } ->>>>>>> origin/master } } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 8faccf7bc7b..3551d58444a 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -349,8 +349,15 @@ static std::tuple executeQueryImpl( { if (!select_with_union_query->list_of_selects->children.empty()) { - if (auto new_settings = select_with_union_query->list_of_selects->children.back()->as()->settings()) - InterpreterSetQuery(new_settings, context).executeForCurrentContext(); + select_with_union_query->dumpTree(std::cerr); + // We might have an arbitrarily complex UNION tree, so just give + // up if the last first-order child is not a plain SELECT. + // It is flattened later, when we process UNION ALL/DISTINCT. + const auto * last_select = select_with_union_query->list_of_selects->children.back()->as(); + if (last_select && last_select->settings()) + { + InterpreterSetQuery(last_select->settings(), context).executeForCurrentContext(); + } } } else if (const auto * query_with_output = dynamic_cast(ast.get())) From 4bf7b54dffac29be4c0fc8daddd4b8b3648ec353 Mon Sep 17 00:00:00 2001 From: feng lv Date: Fri, 6 Nov 2020 11:35:26 +0000 Subject: [PATCH 024/258] fix --- src/Interpreters/executeQuery.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 3551d58444a..1fa28389bb7 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -349,7 +349,6 @@ static std::tuple executeQueryImpl( { if (!select_with_union_query->list_of_selects->children.empty()) { - select_with_union_query->dumpTree(std::cerr); // We might have an arbitrarily complex UNION tree, so just give // up if the last first-order child is not a plain SELECT. // It is flattened later, when we process UNION ALL/DISTINCT. From 125eb0272674f7e086d73faecd7a8c9d4378b279 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sat, 7 Nov 2020 11:38:20 +0000 Subject: [PATCH 025/258] nomalize ASTSelectWithUnionQuery --- .../InterpreterSelectWithUnionQuery.cpp | 25 +++- .../InterpreterSelectWithUnionQuery.h | 2 +- src/Parsers/ASTSelectWithUnionQuery.cpp | 25 ++-- src/Parsers/ASTSelectWithUnionQuery.h | 4 +- src/Parsers/ExpressionListParsers.cpp | 2 +- src/Parsers/ExpressionListParsers.h | 2 +- src/Parsers/ParserSelectWithUnionQuery.cpp | 116 +++++++++++++++- ...t_and_setting_union_default_mode.reference | 23 ---- ..._explain_select_with_union_query.reference | 126 ++++++++++++++++++ .../01556_explain_select_with_union_query.sql | 13 ++ 10 files changed, 284 insertions(+), 54 deletions(-) create mode 100644 tests/queries/0_stateless/01556_explain_select_with_union_query.reference create mode 100644 tests/queries/0_stateless/01556_explain_select_with_union_query.sql diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 7cc0e890fc2..976dcaddd9c 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -170,6 +170,7 @@ Block InterpreterSelectWithUnionQuery::getSampleBlock(const ASTPtr & query_ptr_, return cache[key] = InterpreterSelectWithUnionQuery(query_ptr_, context_, SelectQueryOptions().analyze()).getSampleBlock(); } +#if 0 size_t InterpreterSelectWithUnionQuery::optimizeUnionList() { auto union_distinct_num = 0; @@ -213,10 +214,11 @@ size_t InterpreterSelectWithUnionQuery::optimizeUnionList() } return union_distinct_num; } +#endif void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) { - auto num_distinct_union = optimizeUnionList(); + // auto num_distinct_union = optimizeUnionList(); size_t num_plans = nested_interpreters.size(); /// Skip union for single interpreter. @@ -227,8 +229,8 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) } /// All UNION streams in the chain does not need to do DISTINCT transform - if (num_distinct_union == 0) - { + // if (num_distinct_union == 0) + // { std::vector> plans(num_plans); DataStreams data_streams(num_plans); @@ -243,9 +245,23 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); query_plan.unitePlans(std::move(union_step), std::move(plans)); - } + + const auto & query = query_ptr->as(); + if (query.union_mode == ASTSelectWithUnionQuery::Mode::DISTINCT) + { + /// Add distinct transform + const Settings & settings = context->getSettingsRef(); + SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); + + auto distinct_step + = std::make_unique(query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false); + + query_plan.addStep(std::move(distinct_step)); + } + // } /// The first union_distinct_num UNION streams need to do a DISTINCT transform after unite +#if 0 else { QueryPlan distinct_query_plan; @@ -298,6 +314,7 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) auto final_union_step = std::make_unique(std::move(final_data_streams), result_header, max_threads); query_plan.unitePlans(std::move(final_union_step), std::move(final_plans)); } +#endif } BlockIO InterpreterSelectWithUnionQuery::execute() diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h index 06d31c92a67..9a3035f117c 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -49,7 +49,7 @@ private: std::unique_ptr buildCurrentChildInterpreter(const ASTPtr & ast_ptr_, const Names & current_required_result_column_names); - size_t optimizeUnionList(); + // size_t optimizeUnionList(); }; } diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 639c8ec1b6e..610c82ee03a 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -14,7 +14,7 @@ ASTPtr ASTSelectWithUnionQuery::clone() const res->list_of_selects = list_of_selects->clone(); res->children.push_back(res->list_of_selects); - res->union_modes = union_modes; + res->union_mode = union_mode; cloneOutputOptions(*res); return res; @@ -38,24 +38,15 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) { if (it != list_of_selects->children.begin()) - settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " - << mode_to_str(union_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : ""); + settings.ostr + << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") + << "UNION " + << mode_to_str(union_mode) << (settings.hilite ? hilite_none : ""); if (auto * node = (*it)->as()) { - // just one child in subquery, () is not need - if (node->list_of_selects->children.size() == 1) - { - if (it != list_of_selects->children.begin()) - settings.ostr << settings.nl_or_ws; - node->list_of_selects->children.at(0)->formatImpl(settings, state, frame); - } - // more than one child in subquery - else - { - auto sub_query = std::make_shared(); - sub_query->children.push_back(*it); - sub_query->formatImpl(settings, state, frame); - } + auto sub_query = std::make_shared(); + sub_query->children.push_back(*it); + sub_query->formatImpl(settings, state, frame); } else { diff --git a/src/Parsers/ASTSelectWithUnionQuery.h b/src/Parsers/ASTSelectWithUnionQuery.h index 5600dd4b43a..fd5eeae2d7a 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.h +++ b/src/Parsers/ASTSelectWithUnionQuery.h @@ -23,9 +23,9 @@ public: DISTINCT }; - using Modes = std::vector; + using UnionModes = std::vector; - Modes union_modes; + Mode union_mode; ASTPtr list_of_selects; }; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 1cc72f5fb8b..4eecc9754bf 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -128,7 +128,7 @@ bool ParserUnionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } // SELECT ... UNION SELECT ... else - union_modes.push_back(ASTSelectWithUnionQuery::Mode::Unspecified); + union_modes.push_back(ASTSelectWithUnionQuery::Mode::DISTINCT); return true; } return false; diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index d93952923a9..a239676b7e2 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -119,7 +119,7 @@ private: ParserPtr s_union_parser; ParserPtr s_all_parser; ParserPtr s_distinct_parser; - ASTSelectWithUnionQuery::Modes union_modes; + ASTSelectWithUnionQuery::UnionModes union_modes; }; /** An expression with an infix binary left-associative operator. diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index db586867db0..49f5aa719e5 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -6,6 +6,84 @@ namespace DB { +static void getSelectsFromUnionListNode(ASTPtr & ast_select, ASTs & selects) +{ + if (auto * inner_union = ast_select->as()) + { + /// We need flatten from last to first + for (auto child = inner_union->list_of_selects->children.rbegin(); child != inner_union->list_of_selects->children.rend(); ++child) + getSelectsFromUnionListNode(*child, selects); + + return; + } + + selects.push_back(std::move(ast_select)); +} + +void normalizeSelectList(ASTs & select_list, const ASTSelectWithUnionQuery::UnionModes & union_modes, ASTs & selects) +{ + int i; + for (i = union_modes.size() - 1; i >= 0; --i) + { + if (union_modes[i] == ASTSelectWithUnionQuery::Mode::ALL) + { + if (auto * inner_union = select_list[i + 1]->as()) + { + /// If inner_union is an UNION ALL list, just lift up + if (inner_union->union_mode == ASTSelectWithUnionQuery::Mode::ALL) + { + for (auto child = inner_union->list_of_selects->children.rbegin(); + child != inner_union->list_of_selects->children.rend(); + ++child) + selects.push_back(std::move(*child)); + } + /// inner_union is an UNION DISTINCT list, + // we cann't lift up + else + selects.push_back(std::move(select_list[i + 1])); + } + else + selects.push_back(std::move(select_list[i + 1])); + } + /// flatten all left nodes and current node to a UNION DISTINCT list + else if (union_modes[i] == ASTSelectWithUnionQuery::Mode::DISTINCT) + { + auto distinct_list = std::make_shared(); + distinct_list->list_of_selects = std::make_shared(); + distinct_list->children.push_back(distinct_list->list_of_selects); + for (int j = i + 1; j >= 0; j--) + { + getSelectsFromUnionListNode(select_list[j], distinct_list->list_of_selects->children); + } + distinct_list->union_mode = ASTSelectWithUnionQuery::Mode::DISTINCT; + // Reverse children list + std::reverse(distinct_list->list_of_selects->children.begin(), distinct_list->list_of_selects->children.end()); + selects.push_back(std::move(distinct_list)); + return; + } + } + /// No UNION DISTINCT or only one SELECT in select_list + if (i == -1) + { + if (auto * inner_union = select_list[0]->as()) + { + /// If inner_union is an UNION ALL list, just lift it up + if (inner_union->union_mode == ASTSelectWithUnionQuery::Mode::ALL) + { + for (auto child = inner_union->list_of_selects->children.rbegin(); + child != inner_union->list_of_selects->children.rend(); + ++child) + selects.push_back(std::move(*child)); + } + /// inner_union is an UNION DISTINCT list, + // we cann't lift it up + else + selects.push_back(std::move(select_list[i + 1])); + } + else + selects.push_back(std::move(select_list[0])); + } +} bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { @@ -20,16 +98,44 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & if (!parser.parse(pos, list_node, expected)) return false; + /// NOTE: We cann't simply flatten inner union query now, since we may have different union mode in query, + /// so flatten may change it's semantics. For example: + /// flatten `SELECT 1 UNION (SELECT 1 UNION ALL SELETC 1)` -> `SELECT 1 UNION SELECT 1 UNION ALL SELECT 1` + + /// Before normalize, if we got only one child which is ASTSelectWithUnionQuery, just lift it up + auto & expr_list = list_node->as(); + if (expr_list.children.size() == 1) + { + if (expr_list.children.at(0)->as()) + { + node = std::move(expr_list.children.at(0)); + return true; + } + } + auto select_with_union_query = std::make_shared(); node = select_with_union_query; - select_with_union_query->list_of_selects = list_node; + select_with_union_query->list_of_selects = std::make_shared(); select_with_union_query->children.push_back(select_with_union_query->list_of_selects); - select_with_union_query->union_modes = parser.getUnionModes(); - /// NOTE: We cann't flatten inner union query now, since we may have different union mode in query, - /// so flatten may change it's semantics. For example: - /// flatten `SELECT 1 UNION (SELECT 1 UNION ALL SELETC 1)` -> `SELECT 1 UNION SELECT 1 UNION ALL SELECT 1` + auto union_modes = parser.getUnionModes(); + + normalizeSelectList(expr_list.children, union_modes, select_with_union_query->list_of_selects->children); + /// We need reverse children list + std::reverse(select_with_union_query->list_of_selects->children.begin(), select_with_union_query->list_of_selects->children.end()); + + select_with_union_query->union_mode = ASTSelectWithUnionQuery::Mode::ALL; + + /// After normalize, if we only have one ASTSelectWithUnionQuery child, lift if up + if (select_with_union_query->list_of_selects->children.size() == 1) + { + if (select_with_union_query->list_of_selects->children.at(0)->as()) + { + node = std::move(select_with_union_query->list_of_selects->children.at(0)); + } + } + return true; } diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference index f9f3ee818e9..ff0086583fa 100644 --- a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference @@ -44,16 +44,6 @@ all 1 1 1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -all all all 1 @@ -77,16 +67,3 @@ all 1 1 1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 diff --git a/tests/queries/0_stateless/01556_explain_select_with_union_query.reference b/tests/queries/0_stateless/01556_explain_select_with_union_query.reference new file mode 100644 index 00000000000..c1b07cedd05 --- /dev/null +++ b/tests/queries/0_stateless/01556_explain_select_with_union_query.reference @@ -0,0 +1,126 @@ +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) diff --git a/tests/queries/0_stateless/01556_explain_select_with_union_query.sql b/tests/queries/0_stateless/01556_explain_select_with_union_query.sql new file mode 100644 index 00000000000..abb7f602af5 --- /dev/null +++ b/tests/queries/0_stateless/01556_explain_select_with_union_query.sql @@ -0,0 +1,13 @@ +EXPLAIN SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1; +EXPLAIN (SELECT 1 UNION ALL SELECT 1) UNION ALL SELECT 1; +EXPLAIN SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1); + +EXPLAIN SELECT 1 UNION DISTINCT SELECT 1 UNION DISTINCT SELECT 1; +EXPLAIN (SELECT 1 UNION DISTINCT SELECT 1) UNION DISTINCT SELECT 1; +EXPLAIN SELECT 1 UNION DISTINCT (SELECT 1 UNION DISTINCT SELECT 1); + +EXPLAIN (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1 UNION DISTINCT SELECT 1))) UNION ALL (((SELECT 1) UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1 ) UNION DISTINCT SELECT 1)))); + +EXPLAIN (((((((((((((((SELECT 1 UNION ALL SELECT 1) UNION ALL SELECT 1)))))))))))))); +EXPLAIN (((((((((((((((((((((((((((((SELECT 1 UNION ALL SELECT 1))))))))))))))))))))))))))))); +EXPLAIN (((((((((((((((((((((((((((((SELECT 1 UNION DISTINCT SELECT 1))))))))))))))))))))))))))))); From 26229ed231fc2dcb3dc3e17265197574881d9fff Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 9 Nov 2020 18:07:38 +0300 Subject: [PATCH 026/258] tmp spans for threads (doesn't compile because of json metadata changes) --- src/Common/ThreadStatus.cpp | 1 + src/Common/ThreadStatus.h | 82 +++++++++-------- src/Core/Settings.h | 2 +- src/Formats/FormatSettings.h | 2 + src/Interpreters/InterpreterFactory.cpp | 3 + src/Interpreters/OpenTelemetrySpanLog.cpp | 92 +++++++++++++++++++ src/Interpreters/OpenTelemetrySpanLog.h | 10 ++ src/Interpreters/ThreadStatusExt.cpp | 29 ++++++ src/Interpreters/executeQuery.cpp | 6 +- src/Parsers/parseQuery.cpp | 2 + src/Processors/Executors/PipelineExecutor.cpp | 2 + src/Server/TCPHandler.cpp | 3 + 12 files changed, 195 insertions(+), 39 deletions(-) diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index f5ad28f57af..c7551d42dfe 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 1b4d20e9721..934654cf385 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -31,6 +31,7 @@ class ThreadStatus; class QueryProfilerReal; class QueryProfilerCpu; class QueryThreadLog; +struct OpenTelemetrySpanHolder; class TasksStatsCounters; struct RUsageCounters; struct PerfEventsCounters; @@ -86,9 +87,6 @@ extern thread_local ThreadStatus * current_thread; class ThreadStatus : public boost::noncopyable { public: - ThreadStatus(); - ~ThreadStatus(); - /// Linux's PID (or TGID) (the same id is shown by ps util) const UInt64 thread_id = 0; /// Also called "nice" value. If it was changed to non-zero (when attaching query) - will be reset to zero when query is detached. @@ -110,6 +108,50 @@ public: using Deleter = std::function; Deleter deleter; + __uint128_t opentelemetry_trace_id; + UInt64 opentelemetry_current_span_id; + std::unique_ptr opentelemetry_thread_span; + +protected: + ThreadGroupStatusPtr thread_group; + + std::atomic thread_state{ThreadState::DetachedFromQuery}; + + /// Is set once + Context * global_context = nullptr; + /// Use it only from current thread + Context * query_context = nullptr; + + String query_id; + + /// A logs queue used by TCPHandler to pass logs to a client + InternalTextLogsQueueWeakPtr logs_queue_ptr; + + bool performance_counters_finalized = false; + UInt64 query_start_time_nanoseconds = 0; + UInt64 query_start_time_microseconds = 0; + time_t query_start_time = 0; + size_t queries_started = 0; + + // CPU and Real time query profilers + std::unique_ptr query_profiler_real; + std::unique_ptr query_profiler_cpu; + + Poco::Logger * log = nullptr; + + friend class CurrentThread; + + /// Use ptr not to add extra dependencies in the header + std::unique_ptr last_rusage; + std::unique_ptr taskstats; + + /// Is used to send logs from logs_queue to client in case of fatal errors. + std::function fatal_error_callback; + +public: + ThreadStatus(); + ~ThreadStatus(); + ThreadGroupStatusPtr getThreadGroup() const { return thread_group; @@ -176,40 +218,6 @@ protected: void assertState(const std::initializer_list & permitted_states, const char * description = nullptr) const; - ThreadGroupStatusPtr thread_group; - - std::atomic thread_state{ThreadState::DetachedFromQuery}; - - /// Is set once - Context * global_context = nullptr; - /// Use it only from current thread - Context * query_context = nullptr; - - String query_id; - - /// A logs queue used by TCPHandler to pass logs to a client - InternalTextLogsQueueWeakPtr logs_queue_ptr; - - bool performance_counters_finalized = false; - UInt64 query_start_time_nanoseconds = 0; - UInt64 query_start_time_microseconds = 0; - time_t query_start_time = 0; - size_t queries_started = 0; - - // CPU and Real time query profilers - std::unique_ptr query_profiler_real; - std::unique_ptr query_profiler_cpu; - - Poco::Logger * log = nullptr; - - friend class CurrentThread; - - /// Use ptr not to add extra dependencies in the header - std::unique_ptr last_rusage; - std::unique_ptr taskstats; - - /// Is used to send logs from logs_queue to client in case of fatal errors. - std::function fatal_error_callback; private: void setupState(const ThreadGroupStatusPtr & thread_group_); diff --git a/src/Core/Settings.h b/src/Core/Settings.h index f974a514bc0..71b248edfdd 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -517,7 +517,7 @@ struct Settings : public BaseSettings }; /* - * User-specified file format settings for File and ULR engines. + * User-specified file format settings for File and URL engines. */ DECLARE_SETTINGS_TRAITS(FormatFactorySettingsTraits, FORMAT_FACTORY_SETTINGS) diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index b3c01ddcf14..0a897c96896 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -86,9 +86,11 @@ struct FormatSettings struct { + bool array_of_rows = false; bool quote_64bit_integers = true; bool quote_denormals = true; bool escape_forward_slashes = true; + bool named_tuple_as_object = false; bool serialize_as_strings = false; } json; diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index a18d3ab8a6f..7505c017953 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -66,6 +66,7 @@ #include #include #include +#include #include @@ -93,6 +94,8 @@ namespace ErrorCodes std::unique_ptr InterpreterFactory::get(ASTPtr & query, Context & context, QueryProcessingStage::Enum stage) { + OpenTelemetrySpanHolder span(__FUNCTION__); + ProfileEvents::increment(ProfileEvents::Query); if (query->as()) diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index 6c22165546d..013913d9f2d 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -43,5 +43,97 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(attribute_values); } +OpenTelemetrySpanHolder::OpenTelemetrySpanHolder(const std::string & _operation_name) +{ + auto & thread = CurrentThread::get(); + + trace_id = thread.opentelemetry_trace_id; + if (!trace_id) + { + return; + } + + parent_span_id = thread.opentelemetry_current_span_id; + span_id = thread_local_rng(); + operation_name = _operation_name; + start_time_us = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + + // *** remove this + attribute_names.push_back("start.stacktrace"); + attribute_values.push_back(StackTrace().toString()); + + thread.opentelemetry_current_span_id = span_id; +} + +OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder() +{ + try + { + fmt::print(stderr, "{}\n", StackTrace().toString()); + + if (!trace_id) + { + return; + } + + // First of all, return old value of current span. + auto & thread = CurrentThread::get(); + assert(thread.opentelemetry_current_span_id = span_id); + thread.opentelemetry_current_span_id = parent_span_id; + + // Not sure what's the best way to access the log from here. + auto * thread_group = CurrentThread::getGroup().get(); + // Not sure whether and when this can be null. + if (!thread_group) + { + return; + } + + fmt::print(stderr, "1\n"); + + auto * context = thread_group->query_context; + if (!context) + { + // Both global and query contexts can be null when executing a + // background task, and global context can be null for some + // queries. + return; + } + + //******** remove this + attribute_names.push_back("clickhouse.query_id"); + attribute_values.push_back(context->getCurrentQueryId()); + + fmt::print(stderr, "2\n"); + + auto log = context->getOpenTelemetrySpanLog(); + if (!log) + { + // The log might be disabled. + return; + } + + fmt::print(stderr, "3\n"); + + finish_time_us = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + + // We should use a high resolution monotonic clock for calculating + // duration, but this way will do for now. + duration_ns = (finish_time_us - start_time_us) * 1000; + + + log->add(OpenTelemetrySpanLogElement( + static_cast(*this))); + + fmt::print(stderr, "4\n"); + } + catch (...) + { + tryLogCurrentException(__FUNCTION__); + } +} + } diff --git a/src/Interpreters/OpenTelemetrySpanLog.h b/src/Interpreters/OpenTelemetrySpanLog.h index 271d02804f4..a1c198559b8 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.h +++ b/src/Interpreters/OpenTelemetrySpanLog.h @@ -23,6 +23,10 @@ struct OpenTelemetrySpan struct OpenTelemetrySpanLogElement : public OpenTelemetrySpan { + OpenTelemetrySpanLogElement() = default; + OpenTelemetrySpanLogElement(const OpenTelemetrySpan & span) + : OpenTelemetrySpan(span) {} + static std::string name() { return "OpenTelemetrySpanLog"; } static Block createBlock(); void appendToBlock(MutableColumns & columns) const; @@ -36,4 +40,10 @@ public: using SystemLog::SystemLog; }; +struct OpenTelemetrySpanHolder : public OpenTelemetrySpan +{ + OpenTelemetrySpanHolder(const std::string & _operation_name); + ~OpenTelemetrySpanHolder(); +}; + } diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index adb9a38b10d..1689a1598ba 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -108,8 +109,26 @@ void ThreadStatus::setupState(const ThreadGroupStatusPtr & thread_group_) } if (query_context) + { applyQuerySettings(); + opentelemetry_trace_id = query_context->getClientInfo().opentelemetry_trace_id; + opentelemetry_current_span_id = query_context->getClientInfo().opentelemetry_span_id; + + if (opentelemetry_trace_id) + { + // Register the span for our thread. We might not know the name yet + // -- there are no strong constraints on when it is set relative to + // attaching the thread to query. Will set the name when the span ends. + opentelemetry_thread_span.reset(new OpenTelemetrySpanHolder("")); + } + } + else + { + opentelemetry_trace_id = 0; + opentelemetry_current_span_id = 0; + } + initPerformanceCounters(); thread_state = ThreadState::AttachedToQuery; @@ -300,6 +319,13 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) assertState({ThreadState::AttachedToQuery}, __PRETTY_FUNCTION__); + if (opentelemetry_thread_span) + { + opentelemetry_thread_span->operation_name = getThreadName(); + opentelemetry_thread_span->attribute_names.push_back("clickhouse.thread_id"); + opentelemetry_thread_span->attribute_values.push_back(thread_id); + } + finalizeQueryProfiler(); finalizePerformanceCounters(); @@ -312,6 +338,9 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) query_id.clear(); query_context = nullptr; + opentelemetry_thread_span = nullptr; + opentelemetry_trace_id = 0; + opentelemetry_current_span_id = 0; thread_group.reset(); thread_state = thread_exits ? ThreadState::Died : ThreadState::DetachedFromQuery; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index cdb3d9b7d7b..dbe3d23ca40 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -479,7 +479,11 @@ static std::tuple executeQueryImpl( limits.size_limits = SizeLimits(settings.max_result_rows, settings.max_result_bytes, settings.result_overflow_mode); } - res = interpreter->execute(); + { + OpenTelemetrySpanHolder span("execute interpreter"); + res = interpreter->execute(); + } + QueryPipeline & pipeline = res.pipeline; bool use_processors = pipeline.initialized(); diff --git a/src/Parsers/parseQuery.cpp b/src/Parsers/parseQuery.cpp index f4e4c195506..5ecdd091e11 100644 --- a/src/Parsers/parseQuery.cpp +++ b/src/Parsers/parseQuery.cpp @@ -1,4 +1,6 @@ #include + +#include #include #include #include diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 271903add86..f940ea148c7 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #ifndef NDEBUG #include @@ -75,6 +76,7 @@ static void executeJob(IProcessor * processor) { try { + OpenTelemetrySpanHolder span(demangle(typeid(*processor).name())); processor->work(); } catch (Exception & exception) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 4dceb0aa905..6f484f6df91 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -521,6 +522,8 @@ void TCPHandler::processInsertQuery(const Settings & connection_settings) void TCPHandler::processOrdinaryQuery() { + OpenTelemetrySpanHolder span(__FUNCTION__); + /// Pull query execution result, if exists, and send it to network. if (state.io.in) { From b4c933e585a482a31450252078cadd442310ba12 Mon Sep 17 00:00:00 2001 From: feng lv Date: Mon, 9 Nov 2020 15:44:11 +0000 Subject: [PATCH 027/258] fix fix test --- .../InterpreterSelectWithUnionQuery.cpp | 250 +++++++++--------- .../InterpreterSelectWithUnionQuery.h | 1 - src/Parsers/ASTSelectWithUnionQuery.cpp | 27 +- src/Parsers/ASTSelectWithUnionQuery.h | 4 + src/Parsers/ExpressionListParsers.cpp | 2 +- src/Parsers/ParserSelectWithUnionQuery.cpp | 100 +------ ...t_and_setting_union_default_mode.reference | 40 +++ ...istinct_and_setting_union_default_mode.sql | 33 +++ ..._explain_select_with_union_query.reference | 162 ++++++++++-- .../01556_explain_select_with_union_query.sql | 30 ++- 10 files changed, 396 insertions(+), 253 deletions(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 976dcaddd9c..f12c949eb92 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -11,6 +11,8 @@ #include #include +#include + namespace DB { @@ -21,11 +23,120 @@ namespace ErrorCodes extern const int EXPECTED_ALL_OR_DISTINCT; } +struct CustomizeASTSelectWithUnionQueryNormalize +{ + using TypeToVisit = ASTSelectWithUnionQuery; + + const UnionMode & union_default_mode; + + static void getSelectsFromUnionListNode(ASTPtr & ast_select, ASTs & selects) + { + if (auto * inner_union = ast_select->as()) + { + /// We need flatten from last to first + for (auto child = inner_union->list_of_selects->children.rbegin(); child != inner_union->list_of_selects->children.rend(); + ++child) + getSelectsFromUnionListNode(*child, selects); + + return; + } + + selects.push_back(std::move(ast_select)); + } + + void visit(ASTSelectWithUnionQuery & ast, ASTPtr &) + { + auto & union_modes = ast.list_of_modes; + ASTs selects; + auto & select_list = ast.list_of_selects->children; + + int i; + for (i = union_modes.size() - 1; i >= 0; --i) + { + /// Rewrite UNION Mode + if (union_modes[i] == ASTSelectWithUnionQuery::Mode::Unspecified) + { + if (union_default_mode == UnionMode::ALL) + union_modes[i] = ASTSelectWithUnionQuery::Mode::ALL; + else if (union_default_mode == UnionMode::DISTINCT) + union_modes[i] = ASTSelectWithUnionQuery::Mode::DISTINCT; + else + throw Exception( + "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", + DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); + } + + if (union_modes[i] == ASTSelectWithUnionQuery::Mode::ALL) + { + if (auto * inner_union = select_list[i + 1]->as()) + { + /// Inner_union is an UNION ALL list, just lift up + for (auto child = inner_union->list_of_selects->children.rbegin(); + child != inner_union->list_of_selects->children.rend(); + ++child) + selects.push_back(std::move(*child)); + } + else + selects.push_back(std::move(select_list[i + 1])); + } + /// flatten all left nodes and current node to a UNION DISTINCT list + else if (union_modes[i] == ASTSelectWithUnionQuery::Mode::DISTINCT) + { + auto distinct_list = std::make_shared(); + distinct_list->list_of_selects = std::make_shared(); + distinct_list->children.push_back(distinct_list->list_of_selects); + for (int j = i + 1; j >= 0; j--) + { + getSelectsFromUnionListNode(select_list[j], distinct_list->list_of_selects->children); + } + distinct_list->union_mode = ASTSelectWithUnionQuery::Mode::DISTINCT; + // Reverse children list + std::reverse(distinct_list->list_of_selects->children.begin(), distinct_list->list_of_selects->children.end()); + distinct_list->is_normalized = true; + selects.push_back(std::move(distinct_list)); + break; + } + } + + /// No UNION DISTINCT or only one child in select_list + if (i == -1) + { + if (auto * inner_union = select_list[0]->as()) + { + /// Inner_union is an UNION ALL list, just lift it up + for (auto child = inner_union->list_of_selects->children.rbegin(); child != inner_union->list_of_selects->children.rend(); + ++child) + selects.push_back(std::move(*child)); + } + else + selects.push_back(std::move(select_list[0])); + } + + // reverse children list + std::reverse(selects.begin(), selects.end()); + + ast.is_normalized = true; + ast.union_mode = ASTSelectWithUnionQuery::Mode::ALL; + + ast.list_of_selects->children = std::move(selects); + } +}; + +using CustomizeASTSelectWithUnionQueryNormalizeVisitor + = InDepthNodeVisitor, false>; + InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( const ASTPtr & query_ptr_, const Context & context_, const SelectQueryOptions & options_, const Names & required_result_column_names) : IInterpreterUnionOrSelectQuery(query_ptr_, context_, options_) { - const auto & ast = query_ptr->as(); + auto & ast = query_ptr->as(); + + /// Normalize AST Tree + if (!ast.is_normalized) + { + CustomizeASTSelectWithUnionQueryNormalizeVisitor::Data union_default_mode{context->getSettingsRef().union_default_mode}; + CustomizeASTSelectWithUnionQueryNormalizeVisitor(union_default_mode).visit(query_ptr); + } size_t num_children = ast.list_of_selects->children.size(); if (!num_children) @@ -170,51 +281,6 @@ Block InterpreterSelectWithUnionQuery::getSampleBlock(const ASTPtr & query_ptr_, return cache[key] = InterpreterSelectWithUnionQuery(query_ptr_, context_, SelectQueryOptions().analyze()).getSampleBlock(); } -#if 0 -size_t InterpreterSelectWithUnionQuery::optimizeUnionList() -{ - auto union_distinct_num = 0; - - auto union_default_mode = context->getSettingsRef().union_default_mode; - auto & ast = query_ptr->as(); - size_t num_selects = ast.list_of_selects->children.size(); - - if (!num_selects) - throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); - - if (num_selects > 1) - { - for (auto & mode : ast.union_modes) - { - if (mode == ASTSelectWithUnionQuery::Mode::Unspecified) - { - if (union_default_mode == UnionMode::ALL) - mode = ASTSelectWithUnionQuery::Mode::ALL; - else if (union_default_mode == UnionMode::DISTINCT) - mode = ASTSelectWithUnionQuery::Mode::DISTINCT; - else - throw Exception( - "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", - DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); - } - } - /// Optimize general cases: if there is UNION DISTINCT, all previous UNION DISTINCT can be rewritten to UNION ALL. - /// Therefore we have at most one UNION DISTINCT in a sequence. - for (auto rit = ast.union_modes.rbegin(); rit != ast.union_modes.rend(); ++rit) - { - if (*rit == ASTSelectWithUnionQuery::Mode::DISTINCT) - { - /// Number of streams need to do a DISTINCT transform after unite - union_distinct_num = ast.union_modes.rend() - rit + 1; - for (auto mode_to_modify = ++rit; mode_to_modify != ast.union_modes.rend(); ++mode_to_modify) - *mode_to_modify = ASTSelectWithUnionQuery::Mode::ALL; - break; - } - } - } - return union_distinct_num; -} -#endif void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) { @@ -228,93 +294,33 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) return; } - /// All UNION streams in the chain does not need to do DISTINCT transform - // if (num_distinct_union == 0) - // { - std::vector> plans(num_plans); - DataStreams data_streams(num_plans); + std::vector> plans(num_plans); + DataStreams data_streams(num_plans); - for (size_t i = 0; i < num_plans; ++i) - { - plans[i] = std::make_unique(); - nested_interpreters[i]->buildQueryPlan(*plans[i]); - data_streams[i] = plans[i]->getCurrentDataStream(); - } - - auto max_threads = context->getSettingsRef().max_threads; - auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); - - query_plan.unitePlans(std::move(union_step), std::move(plans)); - - const auto & query = query_ptr->as(); - if (query.union_mode == ASTSelectWithUnionQuery::Mode::DISTINCT) - { - /// Add distinct transform - const Settings & settings = context->getSettingsRef(); - SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); - - auto distinct_step - = std::make_unique(query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false); - - query_plan.addStep(std::move(distinct_step)); - } - // } - - /// The first union_distinct_num UNION streams need to do a DISTINCT transform after unite -#if 0 - else + for (size_t i = 0; i < num_plans; ++i) { - QueryPlan distinct_query_plan; + plans[i] = std::make_unique(); + nested_interpreters[i]->buildQueryPlan(*plans[i]); + data_streams[i] = plans[i]->getCurrentDataStream(); + } - std::vector> plans(num_distinct_union); - DataStreams data_streams(num_distinct_union); + auto max_threads = context->getSettingsRef().max_threads; + auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); - for (size_t i = 0; i < num_distinct_union; ++i) - { - plans[i] = std::make_unique(); - nested_interpreters[i]->buildQueryPlan(*plans[i]); - data_streams[i] = plans[i]->getCurrentDataStream(); - } - - auto max_threads = context->getSettingsRef().max_threads; - auto union_step = std::make_unique(std::move(data_streams), result_header, max_threads); - - distinct_query_plan.unitePlans(std::move(union_step), std::move(plans)); + query_plan.unitePlans(std::move(union_step), std::move(plans)); + const auto & query = query_ptr->as(); + if (query.union_mode == ASTSelectWithUnionQuery::Mode::DISTINCT) + { /// Add distinct transform const Settings & settings = context->getSettingsRef(); SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); - auto distinct_step - = std::make_unique(distinct_query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false); + auto distinct_step = std::make_unique(query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false); - distinct_query_plan.addStep(std::move(distinct_step)); - - /// No other UNION streams after DISTINCT stream - if (num_plans == num_distinct_union) - { - query_plan = std::move(distinct_query_plan); - return; - } - - /// Build final UNION step - std::vector> final_plans(num_plans - num_distinct_union + 1); - DataStreams final_data_streams(num_plans - num_distinct_union + 1); - - final_plans[0] = std::make_unique(std::move(distinct_query_plan)); - final_data_streams[0] = final_plans[0]->getCurrentDataStream(); - - for (size_t i = 1; i < num_plans - num_distinct_union + 1; ++i) - { - final_plans[i] = std::make_unique(); - nested_interpreters[num_distinct_union + i - 1]->buildQueryPlan(*final_plans[i]); - final_data_streams[i] = final_plans[i]->getCurrentDataStream(); - } - - auto final_union_step = std::make_unique(std::move(final_data_streams), result_header, max_threads); - query_plan.unitePlans(std::move(final_union_step), std::move(final_plans)); + query_plan.addStep(std::move(distinct_step)); } -#endif + } BlockIO InterpreterSelectWithUnionQuery::execute() diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h index 9a3035f117c..cd089a51970 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -49,7 +49,6 @@ private: std::unique_ptr buildCurrentChildInterpreter(const ASTPtr & ast_ptr_, const Names & current_required_result_column_names); - // size_t optimizeUnionList(); }; } diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index 610c82ee03a..f51f998efc5 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -3,6 +3,8 @@ #include #include +#include + namespace DB { @@ -16,6 +18,8 @@ ASTPtr ASTSelectWithUnionQuery::clone() const res->union_mode = union_mode; + res->list_of_modes = list_of_modes; + cloneOutputOptions(*res); return res; } @@ -38,15 +42,24 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) { if (it != list_of_selects->children.begin()) - settings.ostr - << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") - << "UNION " - << mode_to_str(union_mode) << (settings.hilite ? hilite_none : ""); + settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION " + << mode_to_str((is_normalized) ? union_mode : list_of_modes[it - list_of_selects->children.begin() - 1]) + << (settings.hilite ? hilite_none : ""); + if (auto * node = (*it)->as()) { - auto sub_query = std::make_shared(); - sub_query->children.push_back(*it); - sub_query->formatImpl(settings, state, frame); + if (node->list_of_selects->children.size() == 1) + { + if (it != list_of_selects->children.begin()) + settings.ostr << settings.nl_or_ws; + (node->list_of_selects->children.at(0))->formatImpl(settings, state, frame); + } + else + { + auto sub_query = std::make_shared(); + sub_query->children.push_back(*it); + sub_query->formatImpl(settings, state, frame); + } } else { diff --git a/src/Parsers/ASTSelectWithUnionQuery.h b/src/Parsers/ASTSelectWithUnionQuery.h index fd5eeae2d7a..ecf03bb6a05 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.h +++ b/src/Parsers/ASTSelectWithUnionQuery.h @@ -27,6 +27,10 @@ public: Mode union_mode; + UnionModes list_of_modes; + + bool is_normalized = false; + ASTPtr list_of_selects; }; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 4eecc9754bf..1cc72f5fb8b 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -128,7 +128,7 @@ bool ParserUnionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } // SELECT ... UNION SELECT ... else - union_modes.push_back(ASTSelectWithUnionQuery::Mode::DISTINCT); + union_modes.push_back(ASTSelectWithUnionQuery::Mode::Unspecified); return true; } return false; diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index 49f5aa719e5..efda8e43ca9 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -6,84 +6,6 @@ namespace DB { -static void getSelectsFromUnionListNode(ASTPtr & ast_select, ASTs & selects) -{ - if (auto * inner_union = ast_select->as()) - { - /// We need flatten from last to first - for (auto child = inner_union->list_of_selects->children.rbegin(); child != inner_union->list_of_selects->children.rend(); ++child) - getSelectsFromUnionListNode(*child, selects); - - return; - } - - selects.push_back(std::move(ast_select)); -} - -void normalizeSelectList(ASTs & select_list, const ASTSelectWithUnionQuery::UnionModes & union_modes, ASTs & selects) -{ - int i; - for (i = union_modes.size() - 1; i >= 0; --i) - { - if (union_modes[i] == ASTSelectWithUnionQuery::Mode::ALL) - { - if (auto * inner_union = select_list[i + 1]->as()) - { - /// If inner_union is an UNION ALL list, just lift up - if (inner_union->union_mode == ASTSelectWithUnionQuery::Mode::ALL) - { - for (auto child = inner_union->list_of_selects->children.rbegin(); - child != inner_union->list_of_selects->children.rend(); - ++child) - selects.push_back(std::move(*child)); - } - /// inner_union is an UNION DISTINCT list, - // we cann't lift up - else - selects.push_back(std::move(select_list[i + 1])); - } - else - selects.push_back(std::move(select_list[i + 1])); - } - /// flatten all left nodes and current node to a UNION DISTINCT list - else if (union_modes[i] == ASTSelectWithUnionQuery::Mode::DISTINCT) - { - auto distinct_list = std::make_shared(); - distinct_list->list_of_selects = std::make_shared(); - distinct_list->children.push_back(distinct_list->list_of_selects); - for (int j = i + 1; j >= 0; j--) - { - getSelectsFromUnionListNode(select_list[j], distinct_list->list_of_selects->children); - } - distinct_list->union_mode = ASTSelectWithUnionQuery::Mode::DISTINCT; - // Reverse children list - std::reverse(distinct_list->list_of_selects->children.begin(), distinct_list->list_of_selects->children.end()); - selects.push_back(std::move(distinct_list)); - return; - } - } - /// No UNION DISTINCT or only one SELECT in select_list - if (i == -1) - { - if (auto * inner_union = select_list[0]->as()) - { - /// If inner_union is an UNION ALL list, just lift it up - if (inner_union->union_mode == ASTSelectWithUnionQuery::Mode::ALL) - { - for (auto child = inner_union->list_of_selects->children.rbegin(); - child != inner_union->list_of_selects->children.rend(); - ++child) - selects.push_back(std::move(*child)); - } - /// inner_union is an UNION DISTINCT list, - // we cann't lift it up - else - selects.push_back(std::move(select_list[i + 1])); - } - else - selects.push_back(std::move(select_list[0])); - } -} bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { @@ -102,7 +24,7 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & /// so flatten may change it's semantics. For example: /// flatten `SELECT 1 UNION (SELECT 1 UNION ALL SELETC 1)` -> `SELECT 1 UNION SELECT 1 UNION ALL SELECT 1` - /// Before normalize, if we got only one child which is ASTSelectWithUnionQuery, just lift it up + /// If we got only one child which is ASTSelectWithUnionQuery, just lift it up auto & expr_list = list_node->as(); if (expr_list.children.size() == 1) { @@ -116,25 +38,9 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & auto select_with_union_query = std::make_shared(); node = select_with_union_query; - select_with_union_query->list_of_selects = std::make_shared(); + select_with_union_query->list_of_selects = list_node; //std::make_shared(); select_with_union_query->children.push_back(select_with_union_query->list_of_selects); - - auto union_modes = parser.getUnionModes(); - - normalizeSelectList(expr_list.children, union_modes, select_with_union_query->list_of_selects->children); - /// We need reverse children list - std::reverse(select_with_union_query->list_of_selects->children.begin(), select_with_union_query->list_of_selects->children.end()); - - select_with_union_query->union_mode = ASTSelectWithUnionQuery::Mode::ALL; - - /// After normalize, if we only have one ASTSelectWithUnionQuery child, lift if up - if (select_with_union_query->list_of_selects->children.size() == 1) - { - if (select_with_union_query->list_of_selects->children.at(0)->as()) - { - node = std::move(select_with_union_query->list_of_selects->children.at(0)); - } - } + select_with_union_query->list_of_modes = parser.getUnionModes(); return true; } diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference index ff0086583fa..9c6ef1adb09 100644 --- a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.reference @@ -8,6 +8,11 @@ 1 1 1 +a +a +a +a +a 1 1 all @@ -44,6 +49,28 @@ all 1 1 1 +1 +1 +1 +1 +1 +a +a +a +a +a +a +a +a +a +a +a +a +1 +1 +1 +1 +all all all 1 @@ -67,3 +94,16 @@ all 1 1 1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql index 6e45c150508..e29e43f64ba 100644 --- a/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql +++ b/tests/queries/0_stateless/01529_union_distinct_and_setting_union_default_mode.sql @@ -1,26 +1,59 @@ SELECT 1; + (((((((SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1; + (((((((SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1; + +SELECT 'a' UNION ALL SELECT 'a' UNION ALL SELECT 'a' UNION SELECT 'a'; + +SELECT 'a' UNION ALL (SELECT 'a' UNION ALL SELECT 'a' UNION SELECT 'a'); + +SELECT 'a' UNION SELECT 'a' UNION SELECT 'a' UNION ALL SELECT'a'; + SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; + SELECT 'all' UNION SELECT 'all' UNION ALL SELECT 'all'; + SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1; + SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; + SELECT 1 UNION (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1 UNION ALL SELECT 1); + SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1 UNION ALL SELECT 1); + SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1))))))); + SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1))))))); + SELECT * FROM (SELECT 1 UNION ALL (SELECT 1 UNION SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1))); SET union_default_mode='ALL'; (((((((SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1) UNION SELECT 1; + (((((((SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1) UNION ALL SELECT 1; + +SELECT 'a' UNION ALL SELECT 'a' UNION ALL SELECT 'a' UNION SELECT 'a'; + +SELECT 'a' UNION ALL (SELECT 'a' UNION ALL SELECT 'a' UNION SELECT 'a'); + +SELECT 'a' UNION SELECT 'a' UNION SELECT 'a' UNION ALL SELECT'a'; + SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; + SELECT 'all' UNION SELECT 'all' UNION ALL SELECT 'all'; + SELECT 1 UNION SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1; + SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION ALL SELECT 1; + SELECT 1 UNION (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1 UNION ALL SELECT 1); + SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1 UNION ALL SELECT 1); + SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1 UNION (SELECT 1))))))); + SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL(SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1))))))); + SELECT * FROM (SELECT 1 UNION ALL (SELECT 1 UNION SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1))); diff --git a/tests/queries/0_stateless/01556_explain_select_with_union_query.reference b/tests/queries/0_stateless/01556_explain_select_with_union_query.reference index c1b07cedd05..cf892c2c591 100644 --- a/tests/queries/0_stateless/01556_explain_select_with_union_query.reference +++ b/tests/queries/0_stateless/01556_explain_select_with_union_query.reference @@ -18,16 +18,17 @@ Union Expression (Projection) Expression (Before ORDER BY and SELECT) ReadFromStorage (Read from SystemOne) -Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) Distinct Union Expression (Projection) @@ -79,6 +80,132 @@ Union Expression (Projection) Expression (Before ORDER BY and SELECT) ReadFromStorage (Read from SystemOne) + Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Distinct + Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) ReadFromStorage (Read from SystemOne) @@ -116,11 +243,10 @@ Union Expression (Projection) Expression (Before ORDER BY and SELECT) ReadFromStorage (Read from SystemOne) -Distinct - Union - Expression (Projection) - Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) - Expression (Projection) - Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) +Union + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) + Expression (Projection) + Expression (Before ORDER BY and SELECT) + ReadFromStorage (Read from SystemOne) diff --git a/tests/queries/0_stateless/01556_explain_select_with_union_query.sql b/tests/queries/0_stateless/01556_explain_select_with_union_query.sql index abb7f602af5..16271113b5f 100644 --- a/tests/queries/0_stateless/01556_explain_select_with_union_query.sql +++ b/tests/queries/0_stateless/01556_explain_select_with_union_query.sql @@ -1,13 +1,29 @@ EXPLAIN SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1; EXPLAIN (SELECT 1 UNION ALL SELECT 1) UNION ALL SELECT 1; -EXPLAIN SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1); +EXPLAIN SELECT 1 UNION (SELECT 1 UNION ALL SELECT 1); -EXPLAIN SELECT 1 UNION DISTINCT SELECT 1 UNION DISTINCT SELECT 1; +EXPLAIN SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1; EXPLAIN (SELECT 1 UNION DISTINCT SELECT 1) UNION DISTINCT SELECT 1; -EXPLAIN SELECT 1 UNION DISTINCT (SELECT 1 UNION DISTINCT SELECT 1); +EXPLAIN SELECT 1 UNION DISTINCT (SELECT 1 UNION SELECT 1); -EXPLAIN (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1 UNION DISTINCT SELECT 1))) UNION ALL (((SELECT 1) UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1 ) UNION DISTINCT SELECT 1)))); +EXPLAIN (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1))) UNION ALL (((SELECT 1) UNION (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION SELECT 1 ) UNION DISTINCT SELECT 1)))); -EXPLAIN (((((((((((((((SELECT 1 UNION ALL SELECT 1) UNION ALL SELECT 1)))))))))))))); -EXPLAIN (((((((((((((((((((((((((((((SELECT 1 UNION ALL SELECT 1))))))))))))))))))))))))))))); -EXPLAIN (((((((((((((((((((((((((((((SELECT 1 UNION DISTINCT SELECT 1))))))))))))))))))))))))))))); +EXPLAIN (((((((((((((((SELECT 1 UNION ALL SELECT 1) UNION SELECT 1)))))))))))))); +EXPLAIN (((((((((((((((((((((((((((((SELECT 1 UNION SELECT 1))))))))))))))))))))))))))))); +EXPLAIN (((((((((((((((((((((((((((((SELECT 1 UNION SELECT 1))))))))))))))))))))))))))))); + +SET union_default_mode='ALL'; + +EXPLAIN SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1; +EXPLAIN (SELECT 1 UNION ALL SELECT 1) UNION ALL SELECT 1; +EXPLAIN SELECT 1 UNION (SELECT 1 UNION ALL SELECT 1); + +EXPLAIN SELECT 1 UNION SELECT 1 UNION DISTINCT SELECT 1; +EXPLAIN (SELECT 1 UNION DISTINCT SELECT 1) UNION DISTINCT SELECT 1; +EXPLAIN SELECT 1 UNION DISTINCT (SELECT 1 UNION SELECT 1); + +EXPLAIN (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION ALL SELECT 1 UNION SELECT 1))) UNION ALL (((SELECT 1) UNION (SELECT 1 UNION ALL (SELECT 1 UNION ALL (SELECT 1 UNION SELECT 1 ) UNION DISTINCT SELECT 1)))); + +EXPLAIN (((((((((((((((SELECT 1 UNION ALL SELECT 1) UNION SELECT 1)))))))))))))); +EXPLAIN (((((((((((((((((((((((((((((SELECT 1 UNION SELECT 1))))))))))))))))))))))))))))); +EXPLAIN (((((((((((((((((((((((((((((SELECT 1 UNION SELECT 1))))))))))))))))))))))))))))); From 81a720d01c7c1241e25f8661dfa64e965e08f595 Mon Sep 17 00:00:00 2001 From: feng lv Date: Tue, 10 Nov 2020 04:28:27 +0000 Subject: [PATCH 028/258] fix special build --- src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index f12c949eb92..936f2138c83 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -44,7 +44,7 @@ struct CustomizeASTSelectWithUnionQueryNormalize selects.push_back(std::move(ast_select)); } - void visit(ASTSelectWithUnionQuery & ast, ASTPtr &) + void visit(ASTSelectWithUnionQuery & ast, ASTPtr &) const { auto & union_modes = ast.list_of_modes; ASTs selects; From 0530c40cd8fdc940e2ad589c08cdb8457ac96641 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 10 Nov 2020 08:50:32 +0300 Subject: [PATCH 029/258] fixes --- src/Common/ThreadStatus.h | 1 - src/Formats/FormatFactory.cpp | 1 + src/Interpreters/OpenTelemetrySpanLog.cpp | 27 +-- src/Interpreters/ThreadStatusExt.cpp | 47 +++-- .../Formats/Impl/JSONRowOutputFormat.cpp | 173 +++++------------- .../Formats/Impl/JSONRowOutputFormat.h | 13 +- 6 files changed, 93 insertions(+), 169 deletions(-) diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 934654cf385..0162a6946c6 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -110,7 +110,6 @@ public: __uint128_t opentelemetry_trace_id; UInt64 opentelemetry_current_span_id; - std::unique_ptr opentelemetry_thread_span; protected: ThreadGroupStatusPtr thread_group; diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 1ff2f0e2a9b..65d1e3ce9fb 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -79,6 +79,7 @@ FormatSettings getFormatSettings(const Context & context, format_settings.input_allow_errors_num = settings.input_format_allow_errors_num; format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio; format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes; + format_settings.json.named_tuple_as_object = settings.output_format_json_named_tuple_as_object; format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; format_settings.null_as_default = settings.input_format_null_as_default; diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index 013913d9f2d..186b067c251 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -40,7 +40,16 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(finish_time_us); columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000)); columns[i++]->insert(attribute_names); - columns[i++]->insert(attribute_values); + // The user might add some ints values, and we will have Int Field, and the + // insert will fail because the column requires Strings. Convert the fields + // here, because it's hard to remember to convert them in all other places. + Array string_values; + string_values.reserve(attribute_values.size()); + for (auto & value : attribute_values) + { + string_values.push_back(toString(value)); + } + columns[i++]->insert(string_values); } OpenTelemetrySpanHolder::OpenTelemetrySpanHolder(const std::string & _operation_name) @@ -59,8 +68,8 @@ OpenTelemetrySpanHolder::OpenTelemetrySpanHolder(const std::string & _operation_ start_time_us = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count(); - // *** remove this - attribute_names.push_back("start.stacktrace"); + // ****** remove this + attribute_names.push_back("clickhouse.start.stacktrace"); attribute_values.push_back(StackTrace().toString()); thread.opentelemetry_current_span_id = span_id; @@ -70,8 +79,6 @@ OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder() { try { - fmt::print(stderr, "{}\n", StackTrace().toString()); - if (!trace_id) { return; @@ -90,8 +97,6 @@ OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder() return; } - fmt::print(stderr, "1\n"); - auto * context = thread_group->query_context; if (!context) { @@ -104,8 +109,8 @@ OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder() //******** remove this attribute_names.push_back("clickhouse.query_id"); attribute_values.push_back(context->getCurrentQueryId()); - - fmt::print(stderr, "2\n"); + attribute_names.push_back("clickhouse.end.stacktrace"); + attribute_values.push_back(StackTrace().toString()); auto log = context->getOpenTelemetrySpanLog(); if (!log) @@ -114,8 +119,6 @@ OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder() return; } - fmt::print(stderr, "3\n"); - finish_time_us = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count(); @@ -126,8 +129,6 @@ OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder() log->add(OpenTelemetrySpanLogElement( static_cast(*this))); - - fmt::print(stderr, "4\n"); } catch (...) { diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 1689a1598ba..3d56182a2f7 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -113,14 +113,13 @@ void ThreadStatus::setupState(const ThreadGroupStatusPtr & thread_group_) applyQuerySettings(); opentelemetry_trace_id = query_context->getClientInfo().opentelemetry_trace_id; - opentelemetry_current_span_id = query_context->getClientInfo().opentelemetry_span_id; - if (opentelemetry_trace_id) { - // Register the span for our thread. We might not know the name yet - // -- there are no strong constraints on when it is set relative to - // attaching the thread to query. Will set the name when the span ends. - opentelemetry_thread_span.reset(new OpenTelemetrySpanHolder("")); + opentelemetry_current_span_id = thread_local_rng(); + } + else + { + opentelemetry_current_span_id = 0; } } else @@ -319,11 +318,38 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) assertState({ThreadState::AttachedToQuery}, __PRETTY_FUNCTION__); - if (opentelemetry_thread_span) + std::shared_ptr opentelemetry_span_log; + if (opentelemetry_trace_id && query_context) { - opentelemetry_thread_span->operation_name = getThreadName(); - opentelemetry_thread_span->attribute_names.push_back("clickhouse.thread_id"); - opentelemetry_thread_span->attribute_values.push_back(thread_id); + opentelemetry_span_log = query_context->getOpenTelemetrySpanLog(); + } + + if (opentelemetry_span_log) + { + // Log the current thread span. + // We do this manually, because we can't use OpenTelemetrySpanHolder as a + // ThreadStatus member, because of linking issues. This file is linked + // separately, so we can reference OpenTelemetrySpanLog here, but if we had + // the span holder as a field, we would have to reference it in the + // destructor, which is in another library. + OpenTelemetrySpanLogElement span; + + span.trace_id = opentelemetry_trace_id; + // Might be problematic if some span holder isn't finished by the time + // we detach this thread... + span.span_id = opentelemetry_current_span_id; + span.parent_span_id = query_context->getClientInfo().opentelemetry_span_id; + span.operation_name = getThreadName(); + span.start_time_us = query_start_time_microseconds; + span.finish_time_us = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + // We could use a more precise and monotonic counter for this. + span.duration_ns = (span.finish_time_us - span.start_time_us) * 1000; + span.attribute_names.push_back("clickhouse.thread_id"); + span.attribute_values.push_back(thread_id); + + opentelemetry_span_log->add(span); } finalizeQueryProfiler(); @@ -338,7 +364,6 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) query_id.clear(); query_context = nullptr; - opentelemetry_thread_span = nullptr; opentelemetry_trace_id = 0; opentelemetry_current_span_id = 0; thread_group.reset(); diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp index babb217ea15..517f126060f 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp @@ -1,50 +1,12 @@ -#include - -#include -#include -#include -#include #include #include +#include #include namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - -void JSONRowOutputFormat::addColumn(String name, DataTypePtr type, - bool & need_validate_utf8, std::string tabs) -{ - if (!type->textCanContainOnlyValidUTF8()) - need_validate_utf8 = true; - - WriteBufferFromOwnString buf; - writeJSONString(name, buf, settings); - - const auto * as_tuple = typeid_cast(type.get()); - const bool recurse = settings.json.named_tuple_as_object - && as_tuple && as_tuple->haveExplicitNames(); - - fields.emplace_back(FieldInfo{buf.str(), type, recurse, tabs}); - - if (recurse) - { - const auto & element_types = as_tuple->getElements(); - const auto & names = as_tuple->getElementNames(); - - assert(element_types.size() == names.size()); - for (size_t i = 0; i < element_types.size(); i++) - { - addColumn(names[i], element_types[i], need_validate_utf8, tabs + "\t"); - } - } -} - JSONRowOutputFormat::JSONRowOutputFormat( WriteBuffer & out_, const Block & header, @@ -55,21 +17,19 @@ JSONRowOutputFormat::JSONRowOutputFormat( { const auto & sample = getPort(PortKind::Main).getHeader(); NamesAndTypesList columns(sample.getNamesAndTypesList()); + fields.assign(columns.begin(), columns.end()); - fields.reserve(columns.size()); - - const std::string initial_tabs = settings.json.write_metadata ? "\t\t\t" : "\t\t"; bool need_validate_utf8 = false; - for (const auto & column : columns) + for (size_t i = 0; i < sample.columns(); ++i) { - addColumn(column.name, column.type, need_validate_utf8, initial_tabs); - } + if (!sample.getByPosition(i).type->textCanContainOnlyValidUTF8()) + need_validate_utf8 = true; -// for (size_t i = 0; i < fields.size(); i++) -// { -// fmt::print(stderr, "{}: '{}' '{}' '{}\n", -// i, fields[i].name, fields[i].type->getName(), fields[i].recurse); -// } + WriteBufferFromOwnString buf; + writeJSONString(fields[i].name, buf, settings); + + fields[i].name = buf.str(); + } if (need_validate_utf8) { @@ -83,76 +43,40 @@ JSONRowOutputFormat::JSONRowOutputFormat( void JSONRowOutputFormat::writePrefix() { - if (settings.json.write_metadata) + writeCString("{\n", *ostr); + writeCString("\t\"meta\":\n", *ostr); + writeCString("\t[\n", *ostr); + + for (size_t i = 0; i < fields.size(); ++i) { - writeCString("{\n", *ostr); - writeCString("\t\"meta\":\n", *ostr); - writeCString("\t[\n", *ostr); + writeCString("\t\t{\n", *ostr); - for (size_t i = 0; i < fields.size(); ++i) - { - writeCString("\t\t{\n", *ostr); - - writeCString("\t\t\t\"name\": ", *ostr); - writeString(fields[i].name, *ostr); - writeCString(",\n", *ostr); - writeCString("\t\t\t\"type\": ", *ostr); - writeJSONString(fields[i].type->getName(), *ostr, settings); - writeChar('\n', *ostr); - - writeCString("\t\t}", *ostr); - if (i + 1 < fields.size()) - writeChar(',', *ostr); - writeChar('\n', *ostr); - } - - writeCString("\t],\n", *ostr); + writeCString("\t\t\t\"name\": ", *ostr); + writeString(fields[i].name, *ostr); + writeCString(",\n", *ostr); + writeCString("\t\t\t\"type\": ", *ostr); + writeJSONString(fields[i].type->getName(), *ostr, settings); + writeChar('\n', *ostr); + + writeCString("\t\t}", *ostr); + if (i + 1 < fields.size()) + writeChar(',', *ostr); writeChar('\n', *ostr); - writeCString("\t\"data\":\n", *ostr); - writeCString("\t", *ostr); } - writeCString("[\n", *ostr); + + writeCString("\t],\n", *ostr); + writeChar('\n', *ostr); + writeCString("\t\"data\":\n", *ostr); + writeCString("\t[\n", *ostr); } + void JSONRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) { -// fmt::print(stderr, "write field column '{}' type '{}'\n", -// column.getName(), type.getName()); - - writeString(fields[field_number].tabs, *ostr); + writeCString("\t\t\t", *ostr); writeString(fields[field_number].name, *ostr); writeCString(": ", *ostr); - // Sanity check: the input column type is the same as in header block. - // If I don't write out the raw pointer explicitly, for some reason clang - // complains about side effect in dereferencing the pointer: - // src/Processors/Formats/Impl/JSONRowOutputFormat.cpp:120:35: warning: expression with side effects will be evaluated despite being used as an operand to 'typeid' [-Wpotentially-evaluated-expression] - [[maybe_unused]] const IDataType * raw_ptr = fields[field_number].type.get(); - assert(typeid(type) == typeid(*raw_ptr)); - - if (fields[field_number].recurse) - { - const auto & tabs = fields[field_number].tabs; - ++field_number; - const auto & tuple_column = assert_cast(column); - const auto & nested_columns = tuple_column.getColumns(); - writeCString("{\n", *ostr); - for (size_t i = 0; i < nested_columns.size(); i++) - { - // field_number is incremented inside, and should match the nested - // columns. - writeField(*nested_columns[i], *fields[field_number].type, row_num); - if (i + 1 < nested_columns.size()) - { - writeCString(",", *ostr); - } - writeCString("\n", *ostr); - } - writeString(tabs, *ostr); - writeCString("}", *ostr); - return; - } - if (yield_strings) { WriteBufferFromOwnString buf; @@ -220,12 +144,6 @@ void JSONRowOutputFormat::writeSuffix() void JSONRowOutputFormat::writeBeforeTotals() { - if (!settings.json.write_metadata) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Cannot output totals in JSON format without metadata"); - } - writeCString(",\n", *ostr); writeChar('\n', *ostr); writeCString("\t\"totals\":\n", *ostr); @@ -254,12 +172,6 @@ void JSONRowOutputFormat::writeAfterTotals() void JSONRowOutputFormat::writeBeforeExtremes() { - if (!settings.json.write_metadata) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Cannot output extremes in JSON format without metadata"); - } - writeCString(",\n", *ostr); writeChar('\n', *ostr); writeCString("\t\"extremes\":\n", *ostr); @@ -305,20 +217,17 @@ void JSONRowOutputFormat::writeAfterExtremes() void JSONRowOutputFormat::writeLastSuffix() { - if (settings.json.write_metadata) - { - writeCString(",\n\n", *ostr); - writeCString("\t\"rows\": ", *ostr); - writeIntText(row_count, *ostr); + writeCString(",\n\n", *ostr); + writeCString("\t\"rows\": ", *ostr); + writeIntText(row_count, *ostr); - writeRowsBeforeLimitAtLeast(); + writeRowsBeforeLimitAtLeast(); - if (settings.write_statistics) - writeStatistics(); + if (settings.write_statistics) + writeStatistics(); - writeChar('\n', *ostr); - writeCString("}\n", *ostr); - } + writeChar('\n', *ostr); + writeCString("}\n", *ostr); ostr->next(); } diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.h b/src/Processors/Formats/Impl/JSONRowOutputFormat.h index a4593663aeb..88b74afbabd 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.h @@ -70,8 +70,6 @@ protected: void writeRowsBeforeLimitAtLeast(); void writeStatistics(); - void addColumn(String name, DataTypePtr type, bool & need_validate_utf8, - std::string tabs); std::unique_ptr validating_ostr; /// Validates UTF-8 sequences, replaces bad sequences with replacement character. WriteBuffer * ostr; @@ -80,16 +78,7 @@ protected: size_t row_count = 0; bool applied_limit = false; size_t rows_before_limit = 0; - - struct FieldInfo - { - String name; - DataTypePtr type; - bool recurse = false; - std::string tabs; - }; - - std::vector fields; + NamesAndTypes fields; Progress progress; Stopwatch watch; From ec2c2ec576f11ed96c0d5072bdd99f5cf90de2f8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 10 Nov 2020 17:09:32 +0300 Subject: [PATCH 030/258] Set expire after we set close --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index f5c57781eef..4df4882a25b 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1114,6 +1114,7 @@ void ZooKeeper::sendThread() info.request->probably_sent = true; info.request->write(*out); + /// We sent close request, exit if (info.request->xid == close_xid) break; } @@ -1342,13 +1343,8 @@ void ZooKeeper::receiveEvent() void ZooKeeper::finalize(bool error_send, bool error_receive) { - { - std::lock_guard lock(push_request_mutex); - - if (expired) - return; - expired = true; - } + if (expired) + return; active_session_metric_increment.destroy(); @@ -1356,7 +1352,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) { if (!error_send) { - /// Send close event. This also signals sending thread to wakeup and then stop. + /// Send close event. This also signals sending thread to stop. try { close(); @@ -1364,12 +1360,22 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) catch (...) { /// This happens for example, when "Cannot push request to queue within operation timeout". + /// Just mark session expired in case of error on close request. + std::lock_guard lock(push_request_mutex); + expired = true; tryLogCurrentException(__PRETTY_FUNCTION__); } + /// Send thread will exit after sending close request or on expired flag send_thread.join(); } + /// Set expired flag after we sent close event + { + std::lock_guard lock(push_request_mutex); + expired = true; + } + try { /// This will also wakeup the receiving thread. From 0f9065a91f94237d56ae63743baaaef2cbd1bcb3 Mon Sep 17 00:00:00 2001 From: feng lv Date: Tue, 10 Nov 2020 06:42:38 +0000 Subject: [PATCH 031/258] fix fix --- src/Interpreters/InterpreterInsertQuery.cpp | 11 +++++++++-- src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 9 +++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index e2830322024..bf8849bd6dc 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -282,10 +282,17 @@ BlockIO InterpreterInsertQuery::execute() if (settings.optimize_trivial_insert_select) { - const auto & selects = query.select->as().list_of_selects->children; + const auto & select_query = query.select->as(); + const auto & selects = select_query.list_of_selects->children; + const auto & union_modes = select_query.list_of_modes; + /// ASTSelectWithUnionQuery is not normalized now, so it may pass some querys which can be Trivial select querys is_trivial_insert_select - = std::all_of(selects.begin(), selects.end(), [](const ASTPtr & select) { return isTrivialSelect(select); }); + = std::all_of( + union_modes.begin(), + union_modes.end(), + [](const ASTSelectWithUnionQuery::Mode & mode) { return mode == ASTSelectWithUnionQuery::Mode::ALL; }) + && std::all_of(selects.begin(), selects.end(), [](const ASTPtr & select) { return isTrivialSelect(select); }); } if (is_trivial_insert_select) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 936f2138c83..1932f49ddc9 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -122,6 +122,7 @@ struct CustomizeASTSelectWithUnionQueryNormalize } }; +/// We need normalize children first, so we should visit AST tree bottom up using CustomizeASTSelectWithUnionQueryNormalizeVisitor = InDepthNodeVisitor, false>; @@ -136,6 +137,14 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( { CustomizeASTSelectWithUnionQueryNormalizeVisitor::Data union_default_mode{context->getSettingsRef().union_default_mode}; CustomizeASTSelectWithUnionQueryNormalizeVisitor(union_default_mode).visit(query_ptr); + + /// After normalization, if it only has one ASTSelectWithUnionQuery child, + /// we can lift it up, this can reduce one unnecessary recursion later. + if (ast.list_of_selects->children.size() == 1 && ast.list_of_selects->children.at(0)->as()) + { + query_ptr = std::move(ast.list_of_selects->children.at(0)); + ast = query_ptr->as(); + } } size_t num_children = ast.list_of_selects->children.size(); From bd3f9e2a22966d6cf92b3b7c6895515e19662119 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 11 Nov 2020 13:09:48 +0300 Subject: [PATCH 032/258] Fix strange code in InterpreterShowAccessQuery --- src/Interpreters/InterpreterShowAccessQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterShowAccessQuery.cpp b/src/Interpreters/InterpreterShowAccessQuery.cpp index 5f28c49c0bc..ecac962878c 100644 --- a/src/Interpreters/InterpreterShowAccessQuery.cpp +++ b/src/Interpreters/InterpreterShowAccessQuery.cpp @@ -78,7 +78,7 @@ ASTs InterpreterShowAccessQuery::getCreateAndGrantQueries() const for (const auto & entity : entities) { create_queries.push_back(InterpreterShowCreateAccessEntityQuery::getCreateQuery(*entity, access_control)); - if (entity->isTypeOf(EntityType::USER) || entity->isTypeOf(EntityType::USER)) + if (entity->isTypeOf(EntityType::USER) || entity->isTypeOf(EntityType::ROLE)) boost::range::push_back(grant_queries, InterpreterShowGrantsQuery::getGrantQueries(*entity, access_control)); } From f04bd0643d3df34b932fa8b3a4770390a0cc9fa2 Mon Sep 17 00:00:00 2001 From: feng lv Date: Wed, 11 Nov 2020 03:45:06 +0000 Subject: [PATCH 033/258] fix test fix fix --- .../InterpreterSelectWithUnionQuery.cpp | 12 +- src/Parsers/ParserSelectWithUnionQuery.cpp | 2 +- ..._explain_select_with_union_query.reference | 216 ++++++++++++------ 3 files changed, 150 insertions(+), 80 deletions(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 1932f49ddc9..a836dc8b271 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -33,10 +33,8 @@ struct CustomizeASTSelectWithUnionQueryNormalize { if (auto * inner_union = ast_select->as()) { - /// We need flatten from last to first - for (auto child = inner_union->list_of_selects->children.rbegin(); child != inner_union->list_of_selects->children.rend(); - ++child) - getSelectsFromUnionListNode(*child, selects); + for (auto & child : inner_union->list_of_selects->children) + getSelectsFromUnionListNode(child, selects); return; } @@ -85,13 +83,13 @@ struct CustomizeASTSelectWithUnionQueryNormalize auto distinct_list = std::make_shared(); distinct_list->list_of_selects = std::make_shared(); distinct_list->children.push_back(distinct_list->list_of_selects); - for (int j = i + 1; j >= 0; j--) + + for (int j = 0; j <= i + 1; ++j) { getSelectsFromUnionListNode(select_list[j], distinct_list->list_of_selects->children); } + distinct_list->union_mode = ASTSelectWithUnionQuery::Mode::DISTINCT; - // Reverse children list - std::reverse(distinct_list->list_of_selects->children.begin(), distinct_list->list_of_selects->children.end()); distinct_list->is_normalized = true; selects.push_back(std::move(distinct_list)); break; diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index efda8e43ca9..9a644d8e937 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -38,7 +38,7 @@ bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & auto select_with_union_query = std::make_shared(); node = select_with_union_query; - select_with_union_query->list_of_selects = list_node; //std::make_shared(); + select_with_union_query->list_of_selects = list_node; select_with_union_query->children.push_back(select_with_union_query->list_of_selects); select_with_union_query->list_of_modes = parser.getUnionModes(); diff --git a/tests/queries/0_stateless/01556_explain_select_with_union_query.reference b/tests/queries/0_stateless/01556_explain_select_with_union_query.reference index cf892c2c591..2d09a1f8625 100644 --- a/tests/queries/0_stateless/01556_explain_select_with_union_query.reference +++ b/tests/queries/0_stateless/01556_explain_select_with_union_query.reference @@ -1,252 +1,324 @@ Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Distinct Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Union Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) Expression (Projection) Expression (Before ORDER BY and SELECT) - ReadFromStorage (Read from SystemOne) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (SystemOne) From 57ac63ca340901e5802ed62a4973f687b7578797 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 11 Nov 2020 19:58:54 +0300 Subject: [PATCH 034/258] Minor change in query profiler --- src/Common/QueryProfiler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index 07e145359d8..504d884dce0 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -176,7 +176,7 @@ template class QueryProfilerBase; template class QueryProfilerBase; QueryProfilerReal::QueryProfilerReal(const UInt64 thread_id, const UInt32 period) - : QueryProfilerBase(thread_id, CLOCK_REALTIME, period, SIGUSR1) + : QueryProfilerBase(thread_id, CLOCK_MONOTONIC, period, SIGUSR1) {} void QueryProfilerReal::signalHandler(int sig, siginfo_t * info, void * context) From 5e1c84e04fc6a681aba3b29111a81f69244243d5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 12 Nov 2020 15:11:13 +0300 Subject: [PATCH 035/258] Fix segfault --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 4df4882a25b..9e20035047d 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1346,8 +1346,6 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) if (expired) return; - active_session_metric_increment.destroy(); - try { if (!error_send) @@ -1376,6 +1374,8 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) expired = true; } + active_session_metric_increment.destroy(); + try { /// This will also wakeup the receiving thread. From 869a6f6aa040af26ba45e6751e47637a1f409c34 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Thu, 12 Nov 2020 16:25:05 +0300 Subject: [PATCH 036/258] Add EN and RU description --- docs/en/operations/settings/settings.md | 10 ++++++++++ docs/ru/operations/settings/settings.md | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f9c3c8a5d75..f720024f524 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2233,4 +2233,14 @@ Possible values: Default value: `1`. +## output_format_tsv_null_representation {#output_format_tsv_null_representation} + +Allows configurable `NULL` representation for [TSV](../../interfaces/formats.md#tabseparated) output format. The setting only controls output format and `\N` is the only supported `NULL` representation for TSV input format. + +Possible values: + +- `\N` — Enabled. + +Default value: `\N`. + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 39a996cb44e..9e192348f51 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2095,4 +2095,14 @@ SELECT CAST(toNullable(toInt32(0)) AS Int32) as x, toTypeName(x); Значение по умолчанию: `1`. +## output_format_tsv_null_representation {#output_format_tsv_null_representation} + +Позволяет настраивать представление `NULL` для формата выходных данных [TSV](../../interfaces/formats.md#tabseparated). Настройка управляет форматом выходных данных, `\N` является единственным поддерживаемым представлением для формата входных данных TSV. + +Возможные значения: + +- `\N` — включено. + +Значение по умолчанию: `\N`. + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) From e19d1430dbb7bb014d95be78db6da5d6949ef95a Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 12 Nov 2020 17:43:16 +0300 Subject: [PATCH 037/258] Fix livelock --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 38 ++++++++++++++++++++------ 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 9e20035047d..addb6c01504 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1070,6 +1070,7 @@ void ZooKeeper::sendThread() setThreadName("ZooKeeperSend"); auto prev_heartbeat_time = clock::now(); + bool tried_to_send_close = false; try { @@ -1099,6 +1100,15 @@ void ZooKeeper::sendThread() std::lock_guard lock(operations_mutex); operations[info.request->xid] = info; } + else + { + /// We set this variable only once. If we will + /// successfully send close, than this thread will just + /// finish. If we will got an exception while sending + /// close, than thread will also finish and finalization + /// will be completed by some other thread. + tried_to_send_close = true; + } if (info.watch) { @@ -1135,7 +1145,13 @@ void ZooKeeper::sendThread() catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); - finalize(true, false); + /// If we have tried to send close and got an exception than + /// finalization is already started by receiveThread and we cannot do + /// anything better than just exit. + /// + /// Otherwise we should correctly finalize + if (!tried_to_send_close) + finalize(true, false); } } @@ -1346,6 +1362,16 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) if (expired) return; + auto expire_session = [&] + { + std::lock_guard lock(push_request_mutex); + if (!expired) + { + expired = true; + active_session_metric_increment.destroy(); + } + }; + try { if (!error_send) @@ -1359,8 +1385,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) { /// This happens for example, when "Cannot push request to queue within operation timeout". /// Just mark session expired in case of error on close request. - std::lock_guard lock(push_request_mutex); - expired = true; + expire_session(); tryLogCurrentException(__PRETTY_FUNCTION__); } @@ -1369,12 +1394,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) } /// Set expired flag after we sent close event - { - std::lock_guard lock(push_request_mutex); - expired = true; - } - - active_session_metric_increment.destroy(); + expire_session(); try { From bf68f35ee6f0a8f4d1de7f46933c8606128f38ef Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 13 Nov 2020 08:59:50 +0100 Subject: [PATCH 038/258] RBAC Testflows tests for ATTACH, CREATE, DROP, and DETACH --- tests/testflows/rbac/helper/common.py | 5 + tests/testflows/rbac/regression.py | 6 + .../rbac/requirements/requirements.md | 349 ++++++-- .../rbac/requirements/requirements.py | 782 +++++++++++++++--- .../tests/privileges/alter/alter_fetch.py | 9 + .../rbac/tests/privileges/alter/alter_move.py | 56 +- .../rbac/tests/privileges/attach/__init__.py | 0 .../privileges/attach/attach_database.py | 99 +++ .../privileges/attach/attach_dictionary.py | 99 +++ .../tests/privileges/attach/attach_table.py | 99 +++ .../privileges/attach/attach_temp_table.py | 97 +++ .../rbac/tests/privileges/create/__init__.py | 0 .../privileges/create/create_database.py | 98 +++ .../privileges/create/create_dictionary.py | 98 +++ .../tests/privileges/create/create_table.py | 731 ++++++++++++++++ .../privileges/create/create_temp_table.py | 98 +++ .../rbac/tests/privileges/detach/__init__.py | 0 .../privileges/detach/detach_database.py | 106 +++ .../privileges/detach/detach_dictionary.py | 105 +++ .../tests/privileges/detach/detach_table.py | 107 +++ .../tests/privileges/detach/detach_view.py | 107 +++ .../tests/privileges/distributed_table.py | 26 +- .../rbac/tests/privileges/drop/__init__.py | 0 .../tests/privileges/drop/drop_database.py | 106 +++ .../tests/privileges/drop/drop_dictionary.py | 102 +++ .../rbac/tests/privileges/drop/drop_table.py | 106 +++ .../rbac/tests/privileges/feature.py | 22 +- 27 files changed, 3221 insertions(+), 192 deletions(-) create mode 100644 tests/testflows/rbac/tests/privileges/attach/__init__.py create mode 100644 tests/testflows/rbac/tests/privileges/attach/attach_database.py create mode 100644 tests/testflows/rbac/tests/privileges/attach/attach_dictionary.py create mode 100644 tests/testflows/rbac/tests/privileges/attach/attach_table.py create mode 100644 tests/testflows/rbac/tests/privileges/attach/attach_temp_table.py create mode 100644 tests/testflows/rbac/tests/privileges/create/__init__.py create mode 100644 tests/testflows/rbac/tests/privileges/create/create_database.py create mode 100644 tests/testflows/rbac/tests/privileges/create/create_dictionary.py create mode 100644 tests/testflows/rbac/tests/privileges/create/create_table.py create mode 100644 tests/testflows/rbac/tests/privileges/create/create_temp_table.py create mode 100644 tests/testflows/rbac/tests/privileges/detach/__init__.py create mode 100644 tests/testflows/rbac/tests/privileges/detach/detach_database.py create mode 100644 tests/testflows/rbac/tests/privileges/detach/detach_dictionary.py create mode 100644 tests/testflows/rbac/tests/privileges/detach/detach_table.py create mode 100644 tests/testflows/rbac/tests/privileges/detach/detach_view.py create mode 100644 tests/testflows/rbac/tests/privileges/drop/__init__.py create mode 100644 tests/testflows/rbac/tests/privileges/drop/drop_database.py create mode 100644 tests/testflows/rbac/tests/privileges/drop/drop_dictionary.py create mode 100644 tests/testflows/rbac/tests/privileges/drop/drop_table.py diff --git a/tests/testflows/rbac/helper/common.py b/tests/testflows/rbac/helper/common.py index 19bc396a0b8..47e38560714 100755 --- a/tests/testflows/rbac/helper/common.py +++ b/tests/testflows/rbac/helper/common.py @@ -149,11 +149,16 @@ def grant_select_on_table(node, grants, target_name, *table_names): try: tables_granted = [] for table_number in range(len(table_names)): + if(grants & tables[f"table{table_number}"]): + with When(f"I grant select privilege on {table_names[table_number]}"): node.query(f"GRANT SELECT ON {table_names[table_number]} TO {target_name}") + tables_granted.append(f'{table_names[table_number]}') + yield (', ').join(tables_granted) + finally: for table_number in range(len(table_names)): with Finally(f"I revoke the select privilege on {table_names[table_number]}"): diff --git a/tests/testflows/rbac/regression.py b/tests/testflows/rbac/regression.py index a53de0178eb..c34e25ce918 100755 --- a/tests/testflows/rbac/regression.py +++ b/tests/testflows/rbac/regression.py @@ -96,6 +96,12 @@ xfails = { [(Fail, issue_16403)], "privileges/alter move/:/:/:/:/user with revoked ALTER MOVE PARTITION privilege/": [(Fail, issue_16403)], + "/rbac/privileges/create table/create with join query privilege granted directly or via role/:": + [(Fail, issue_14149)], + "/rbac/privileges/create table/create with join union subquery privilege granted directly or via role/:": + [(Fail, issue_14149)], + "/rbac/privileges/create table/create with nested tables privilege granted directly or via role/:": + [(Fail, issue_14149)], } xflags = { diff --git a/tests/testflows/rbac/requirements/requirements.md b/tests/testflows/rbac/requirements/requirements.md index 4f2be6776c0..933745838e0 100644 --- a/tests/testflows/rbac/requirements/requirements.md +++ b/tests/testflows/rbac/requirements/requirements.md @@ -386,32 +386,32 @@ * 5.2.11.3.4 [RQ.SRS-006.RBAC.Privileges.Insert.Column](#rqsrs-006rbacprivilegesinsertcolumn) * 5.2.11.3.5 [RQ.SRS-006.RBAC.Privileges.Insert.Cluster](#rqsrs-006rbacprivilegesinsertcluster) * 5.2.11.3.6 [RQ.SRS-006.RBAC.Privileges.Insert.TableEngines](#rqsrs-006rbacprivilegesinserttableengines) - * 5.2.11.4 [AlterColumn](#altercolumn) + * 5.2.11.4 [Alter Column](#alter-column) * 5.2.11.4.1 [RQ.SRS-006.RBAC.Privileges.AlterColumn](#rqsrs-006rbacprivilegesaltercolumn) * 5.2.11.4.2 [RQ.SRS-006.RBAC.Privileges.AlterColumn.Grant](#rqsrs-006rbacprivilegesaltercolumngrant) * 5.2.11.4.3 [RQ.SRS-006.RBAC.Privileges.AlterColumn.Revoke](#rqsrs-006rbacprivilegesaltercolumnrevoke) * 5.2.11.4.4 [RQ.SRS-006.RBAC.Privileges.AlterColumn.Column](#rqsrs-006rbacprivilegesaltercolumncolumn) * 5.2.11.4.5 [RQ.SRS-006.RBAC.Privileges.AlterColumn.Cluster](#rqsrs-006rbacprivilegesaltercolumncluster) * 5.2.11.4.6 [RQ.SRS-006.RBAC.Privileges.AlterColumn.TableEngines](#rqsrs-006rbacprivilegesaltercolumntableengines) - * 5.2.11.5 [AlterIndex](#alterindex) + * 5.2.11.5 [Alter Index](#alter-index) * 5.2.11.5.1 [RQ.SRS-006.RBAC.Privileges.AlterIndex](#rqsrs-006rbacprivilegesalterindex) * 5.2.11.5.2 [RQ.SRS-006.RBAC.Privileges.AlterIndex.Grant](#rqsrs-006rbacprivilegesalterindexgrant) * 5.2.11.5.3 [RQ.SRS-006.RBAC.Privileges.AlterIndex.Revoke](#rqsrs-006rbacprivilegesalterindexrevoke) * 5.2.11.5.4 [RQ.SRS-006.RBAC.Privileges.AlterIndex.Cluster](#rqsrs-006rbacprivilegesalterindexcluster) * 5.2.11.5.5 [RQ.SRS-006.RBAC.Privileges.AlterIndex.TableEngines](#rqsrs-006rbacprivilegesalterindextableengines) - * 5.2.11.6 [AlterConstraint](#alterconstraint) + * 5.2.11.6 [Alter Constraint](#alter-constraint) * 5.2.11.6.1 [RQ.SRS-006.RBAC.Privileges.AlterConstraint](#rqsrs-006rbacprivilegesalterconstraint) * 5.2.11.6.2 [RQ.SRS-006.RBAC.Privileges.AlterConstraint.Grant](#rqsrs-006rbacprivilegesalterconstraintgrant) * 5.2.11.6.3 [RQ.SRS-006.RBAC.Privileges.AlterConstraint.Revoke](#rqsrs-006rbacprivilegesalterconstraintrevoke) * 5.2.11.6.4 [RQ.SRS-006.RBAC.Privileges.AlterConstraint.Cluster](#rqsrs-006rbacprivilegesalterconstraintcluster) * 5.2.11.6.5 [RQ.SRS-006.RBAC.Privileges.AlterConstraint.TableEngines](#rqsrs-006rbacprivilegesalterconstrainttableengines) - * 5.2.11.7 [AlterTTL](#alterttl) + * 5.2.11.7 [Alter TTL](#alter-ttl) * 5.2.11.7.1 [RQ.SRS-006.RBAC.Privileges.AlterTTL](#rqsrs-006rbacprivilegesalterttl) * 5.2.11.7.2 [RQ.SRS-006.RBAC.Privileges.AlterTTL.Grant](#rqsrs-006rbacprivilegesalterttlgrant) * 5.2.11.7.3 [RQ.SRS-006.RBAC.Privileges.AlterTTL.Revoke](#rqsrs-006rbacprivilegesalterttlrevoke) * 5.2.11.7.4 [RQ.SRS-006.RBAC.Privileges.AlterTTL.Cluster](#rqsrs-006rbacprivilegesalterttlcluster) * 5.2.11.7.5 [RQ.SRS-006.RBAC.Privileges.AlterTTL.TableEngines](#rqsrs-006rbacprivilegesalterttltableengines) - * 5.2.11.8 [AlterSettings](#altersettings) + * 5.2.11.8 [Alter Settings](#alter-settings) * 5.2.11.8.1 [RQ.SRS-006.RBAC.Privileges.AlterSettings](#rqsrs-006rbacprivilegesaltersettings) * 5.2.11.8.2 [RQ.SRS-006.RBAC.Privileges.AlterSettings.Grant](#rqsrs-006rbacprivilegesaltersettingsgrant) * 5.2.11.8.3 [RQ.SRS-006.RBAC.Privileges.AlterSettings.Revoke](#rqsrs-006rbacprivilegesaltersettingsrevoke) @@ -437,15 +437,55 @@ * 5.2.11.13.1 [RQ.SRS-006.RBAC.Privileges.AlterMove](#rqsrs-006rbacprivilegesaltermove) * 5.2.11.13.2 [RQ.SRS-006.RBAC.Privileges.AlterMove.Access](#rqsrs-006rbacprivilegesaltermoveaccess) * 5.2.11.13.3 [RQ.SRS-006.RBAC.Privileges.AlterMove.TableEngines](#rqsrs-006rbacprivilegesaltermovetableengines) - * 5.2.11.14 [Grant Option](#grant-option) - * 5.2.11.14.1 [RQ.SRS-006.RBAC.Privileges.GrantOption](#rqsrs-006rbacprivilegesgrantoption) - * 5.2.11.15 [RQ.SRS-006.RBAC.Privileges.Delete](#rqsrs-006rbacprivilegesdelete) - * 5.2.11.16 [RQ.SRS-006.RBAC.Privileges.Alter](#rqsrs-006rbacprivilegesalter) - * 5.2.11.17 [RQ.SRS-006.RBAC.Privileges.Create](#rqsrs-006rbacprivilegescreate) - * 5.2.11.18 [RQ.SRS-006.RBAC.Privileges.Drop](#rqsrs-006rbacprivilegesdrop) - * 5.2.11.19 [RQ.SRS-006.RBAC.Privileges.All](#rqsrs-006rbacprivilegesall) - * 5.2.11.20 [RQ.SRS-006.RBAC.Privileges.All.GrantRevoke](#rqsrs-006rbacprivilegesallgrantrevoke) - * 5.2.11.21 [RQ.SRS-006.RBAC.Privileges.AdminOption](#rqsrs-006rbacprivilegesadminoption) + * 5.2.11.14 [Create Table](#create-table) + * 5.2.11.14.1 [RQ.SRS-006.RBAC.Privileges.CreateTable](#rqsrs-006rbaccreatetable) + * 5.2.11.14.2 [RQ.SRS-006.RBAC.Privileges.CreateTable.Access](#rqsrs-006rbacprivilegescreatetableaccess) + * 5.2.11.15 [Create Database](#create-database) + * 5.2.11.15.1 [RQ.SRS-006.RBAC.Privileges.CreateDatabase](#rqsrs-006rbacprivilegescreatedatabase) + * 5.2.11.15.2 [RQ.SRS-006.RBAC.Privileges.CreateDatabase.Access](#rqsrs-006rbacprivilegescreatedatabaseaccess) + * 5.2.11.16 [Create Dictionary](#create-dictionary) + * 5.2.11.16.1 [RQ.SRS-006.RBAC.Privileges.CreateDictionary](#rqsrs-006rbacprivilegescreatedictionary) + * 5.2.11.16.2 [RQ.SRS-006.RBAC.Privileges.CreateDictionary.Access](#rqsrs-006rbacprivilegescreatedictionaryaccess) + * 5.2.11.17 [Create Temporary Table](#create-temporary-table) + * 5.2.11.17.1 [RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable](#rqsrs-006rbacprivilegescreatetemporarytable) + * 5.2.11.17.2 [RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable.Access](#rqsrs-006rbacprivilegescreatetemporarytableaccess) + * 5.2.11.18 [Attach Database](#attach-database) + * 5.2.11.18.1 [RQ.SRS-006.RBAC.Privileges.AttachDatabase](#rqsrs-006rbacprivilegesattachdatabase) + * 5.2.11.19 [Attach Dictionary](#attach-dictionary) + * 5.2.11.19.1 [RQ.SRS-006.RBAC.Privileges.AttachDictionary](#rqsrs-006rbacprivilegesattachdictionary) + * 5.2.11.20 [Attach Temporary Table](#attach-temporary-table) + * 5.2.11.20.1 [RQ.SRS-006.RBAC.Privileges.AttachTemporaryTable](#rqsrs-006rbacprivilegesattachtemporarytable) + * 5.2.11.21 [Attach Table](#attach-table) + * 5.2.11.21.1 [RQ.SRS-006.RBAC.Privileges.AttachTable](#rqsrs-006rbacprivilegesattachtable) + * 5.2.11.22 [Drop Table](#drop-table) + * 5.2.11.22.1 [RQ.SRS-006.RBAC.Privileges.DropTable](#rqsrs-006rbacprivilegesdroptable) + * 5.2.11.22.2 [RQ.SRS-006.RBAC.Privileges.DropTable.Access](#rqsrs-006rbacprivilegesdroptableaccess) + * 5.2.11.23 [Drop View](#drop-view) + * 5.2.11.23.1 [RQ.SRS-006.RBAC.Privileges.DropView](#rqsrs-006rbacprivilegesdropview) + * 5.2.11.23.2 [RQ.SRS-006.RBAC.Privileges.DropView.Access](#rqsrs-006rbacprivilegesdropviewaccess) + * 5.2.11.24 [Drop Database](#drop-database) + * 5.2.11.24.1 [RQ.SRS-006.RBAC.Privileges.DropDatabase](#rqsrs-006rbacprivilegesdropdatabase) + * 5.2.11.24.2 [RQ.SRS-006.RBAC.Privileges.DropDatabase.Access](#rqsrs-006rbacprivilegesdropdatabaseaccess) + * 5.2.11.25 [Drop Dictionary](#drop-dictionary) + * 5.2.11.25.1 [RQ.SRS-006.RBAC.Privileges.DropDictionary](#rqsrs-006rbacprivilegesdropdictionary) + * 5.2.11.25.2 [RQ.SRS-006.RBAC.Privileges.DropDictionary.Access](#rqsrs-006rbacprivilegesdropdictionaryaccess) + * 5.2.11.26 [Detach Table](#detach-table) + * 5.2.11.26.1 [RQ.SRS-006.RBAC.Privileges.DetachTable](#rqsrs-006rbacprivilegesdetachtable) + * 5.2.11.27 [Detach View](#detach-view) + * 5.2.11.27.1 [RQ.SRS-006.RBAC.Privileges.DetachView](#rqsrs-006rbacprivilegesdetachview) + * 5.2.11.28 [Detach Database](#detach-database) + * 5.2.11.28.1 [RQ.SRS-006.RBAC.Privileges.DetachDatabase](#rqsrs-006rbacprivilegesdetachdatabase) + * 5.2.11.29 [Detach Dictionary](#detach-dictionary) + * 5.2.11.29.1 [RQ.SRS-006.RBAC.Privileges.DetachDictionary](#rqsrs-006rbacprivilegesdetachdictionary) + * 5.2.11.30 [Grant Option](#grant-option) + * 5.2.11.30.1 [RQ.SRS-006.RBAC.Privileges.GrantOption](#rqsrs-006rbacprivilegesgrantoption) + * 5.2.11.31 [RQ.SRS-006.RBAC.Privileges.Delete](#rqsrs-006rbacprivilegesdelete) + * 5.2.11.32 [RQ.SRS-006.RBAC.Privileges.Alter](#rqsrs-006rbacprivilegesalter) + * 5.2.11.33 [RQ.SRS-006.RBAC.Privileges.Create](#rqsrs-006rbacprivilegescreate) + * 5.2.11.34 [RQ.SRS-006.RBAC.Privileges.Drop](#rqsrs-006rbacprivilegesdrop) + * 5.2.11.35 [RQ.SRS-006.RBAC.Privileges.All](#rqsrs-006rbacprivilegesall) + * 5.2.11.36 [RQ.SRS-006.RBAC.Privileges.All.GrantRevoke](#rqsrs-006rbacprivilegesallgrantrevoke) + * 5.2.11.37 [RQ.SRS-006.RBAC.Privileges.AdminOption](#rqsrs-006rbacprivilegesadminoption) * 5.2.12 [Required Privileges](#required-privileges) * 5.2.12.1 [RQ.SRS-006.RBAC.RequiredPrivileges.Create](#rqsrs-006rbacrequiredprivilegescreate) * 5.2.12.2 [RQ.SRS-006.RBAC.RequiredPrivileges.Alter](#rqsrs-006rbacrequiredprivilegesalter) @@ -1027,7 +1067,7 @@ to some password as identification when altering user account using ##### RQ.SRS-006.RBAC.User.Alter.Host.AddDrop version: 1.0 -[ClickHouse] SHALL support altering user by adding and dropping access to hosts with the `ADD HOST` or the `DROP HOST`in the `ALTER USER` statement. +[ClickHouse] SHALL support altering user by adding and dropping access to hosts with the `ADD HOST` or the `CREATE HOST`in the `ALTER USER` statement. ##### RQ.SRS-006.RBAC.User.Alter.Host.Local version: 1.0 @@ -1122,7 +1162,7 @@ version: 1.0 ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] [RENAME TO new_name] [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] - [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [[ADD|CREATE] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] ``` @@ -1231,12 +1271,12 @@ SHOW CREATE USER [name | CURRENT_USER] ##### RQ.SRS-006.RBAC.User.Drop version: 1.0 -[ClickHouse] SHALL support removing a user account using `DROP USER` statement. +[ClickHouse] SHALL support removing a user account using `CREATE USER` statement. ##### RQ.SRS-006.RBAC.User.Drop.IfExists version: 1.0 -[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP USER` statement +[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE USER` statement to skip raising an exception if the user account does not exist. If the `IF EXISTS` clause is not specified then an exception SHALL be raised if a user does not exist. @@ -1244,16 +1284,16 @@ raised if a user does not exist. ##### RQ.SRS-006.RBAC.User.Drop.OnCluster version: 1.0 -[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP USER` statement +[ClickHouse] SHALL support using `ON CLUSTER` clause in the `CREATE USER` statement to specify the name of the cluster the user should be dropped from. ##### RQ.SRS-006.RBAC.User.Drop.Syntax version: 1.0 -[ClickHouse] SHALL support the following syntax for `DROP USER` statement +[ClickHouse] SHALL support the following syntax for `CREATE USER` statement ```sql -DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +CREATE USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` ##### RQ.SRS-006.RBAC.Role.Create @@ -1340,12 +1380,12 @@ ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] ##### RQ.SRS-006.RBAC.Role.Drop version: 1.0 -[ClickHouse] SHALL support removing one or more roles using `DROP ROLE` statement. +[ClickHouse] SHALL support removing one or more roles using `CREATE ROLE` statement. ##### RQ.SRS-006.RBAC.Role.Drop.IfExists version: 1.0 -[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP ROLE` statement +[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE ROLE` statement to skip raising an exception if the role does not exist. If the `IF EXISTS` clause is not specified then an exception SHALL be raised if a role does not exist. @@ -1353,15 +1393,15 @@ raised if a role does not exist. ##### RQ.SRS-006.RBAC.Role.Drop.Cluster version: 1.0 -[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP ROLE` statement to specify the cluster from which to drop the specified role. +[ClickHouse] SHALL support using `ON CLUSTER` clause in the `CREATE ROLE` statement to specify the cluster from which to drop the specified role. ##### RQ.SRS-006.RBAC.Role.Drop.Syntax version: 1.0 -[ClickHouse] SHALL support the following syntax for the `DROP ROLE` statement +[ClickHouse] SHALL support the following syntax for the `CREATE ROLE` statement ``` sql -DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +CREATE ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` ##### RQ.SRS-006.RBAC.Role.ShowCreate @@ -1419,7 +1459,7 @@ for a database or a table using the `GRANT CREATE` statement. version: 1.0 [ClickHouse] SHALL support granting the **drop** privilege to one or more users or roles -for a database or a table using the `GRANT DROP` statement. +for a database or a table using the `GRANT CREATE` statement. ##### RQ.SRS-006.RBAC.Grant.Privilege.Truncate version: 1.0 @@ -1545,7 +1585,7 @@ version: 1.0 [ClickHouse] SHALL support revoking ANY privilege to one or more users or roles for a database or a table using the `REVOKE some_privilege` statement. **some_privilege** refers to any Clickhouse defined privilege, whose hierarchy includes -SELECT, INSERT, ALTER, CREATE, DROP, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT, +SELECT, INSERT, ALTER, CREATE, CREATE, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT, SYSTEM, INTROSPECTION, SOURCES, dictGet and all of their sub-privileges. ##### RQ.SRS-006.RBAC.Revoke.Privilege.Select @@ -1576,7 +1616,7 @@ for a database or a table using the `REVOKE CREATE` statement. version: 1.0 [ClickHouse] SHALL support revoking the **drop** privilege to one or more users or roles -for a database or a table using the `REVOKE DROP` statement. +for a database or a table using the `REVOKE CREATE` statement. ##### RQ.SRS-006.RBAC.Revoke.Privilege.Truncate version: 1.0 @@ -1646,7 +1686,7 @@ version: 1.0 [ClickHouse] SHALL support revoking MULTIPLE **privileges** to one or more users or roles for a database or a table using the `REVOKE privilege1, privilege2...` statement. **privileges** refers to any set of Clickhouse defined privilege, whose hierarchy includes -SELECT, INSERT, ALTER, CREATE, DROP, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT, +SELECT, INSERT, ALTER, CREATE, CREATE, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT, SYSTEM, INTROSPECTION, SOURCES, dictGet and all of their sub-privileges. ##### RQ.SRS-006.RBAC.Revoke.Privilege.All @@ -1985,12 +2025,12 @@ ALTER SETTINGS PROFILE [IF EXISTS] name ##### RQ.SRS-006.RBAC.SettingsProfile.Drop version: 1.0 -[ClickHouse] SHALL support removing one or more settings profiles using the `DROP SETTINGS PROFILE` statement. +[ClickHouse] SHALL support removing one or more settings profiles using the `CREATE SETTINGS PROFILE` statement. ##### RQ.SRS-006.RBAC.SettingsProfile.Drop.IfExists version: 1.0 -[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP SETTINGS PROFILE` statement +[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE SETTINGS PROFILE` statement to skip raising an exception if the settings profile does not exist. If the `IF EXISTS` clause is not specified then an exception SHALL be raised if a settings profile does not exist. @@ -1999,15 +2039,15 @@ raised if a settings profile does not exist. version: 1.0 [ClickHouse] SHALL support dropping one or more settings profiles on specified cluster using -`ON CLUSTER` clause in the `DROP SETTINGS PROFILE` statement. +`ON CLUSTER` clause in the `CREATE SETTINGS PROFILE` statement. ##### RQ.SRS-006.RBAC.SettingsProfile.Drop.Syntax version: 1.0 -[ClickHouse] SHALL support the following syntax for the `DROP SETTINGS PROFILE` statement +[ClickHouse] SHALL support the following syntax for the `CREATE SETTINGS PROFILE` statement ``` sql -DROP SETTINGS PROFILE [IF EXISTS] name [,name,...] +CREATE SETTINGS PROFILE [IF EXISTS] name [,name,...] ``` ##### RQ.SRS-006.RBAC.SettingsProfile.ShowCreateSettingsProfile @@ -2342,12 +2382,12 @@ ALTER QUOTA [IF EXIST] name ##### RQ.SRS-006.RBAC.Quota.Drop version: 1.0 -[ClickHouse] SHALL support removing one or more quotas using the `DROP QUOTA` statement. +[ClickHouse] SHALL support removing one or more quotas using the `CREATE QUOTA` statement. ##### RQ.SRS-006.RBAC.Quota.Drop.IfExists version: 1.0 -[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP QUOTA` statement +[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE QUOTA` statement to skip raising an exception when the quota does not exist. If the `IF EXISTS` clause is not specified then an exception SHALL be raised if the quota does not exist. @@ -2355,16 +2395,16 @@ raised if the quota does not exist. ##### RQ.SRS-006.RBAC.Quota.Drop.Cluster version: 1.0 -[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP QUOTA` statement +[ClickHouse] SHALL support using `ON CLUSTER` clause in the `CREATE QUOTA` statement to indicate the cluster the quota to be dropped is located on. ##### RQ.SRS-006.RBAC.Quota.Drop.Syntax version: 1.0 -[ClickHouse] SHALL support the following syntax for the `DROP QUOTA` statement +[ClickHouse] SHALL support the following syntax for the `CREATE QUOTA` statement ``` sql -DROP QUOTA [IF EXISTS] name [,name...] +CREATE QUOTA [IF EXISTS] name [,name...] ``` ##### RQ.SRS-006.RBAC.Quota.ShowQuotas @@ -2636,12 +2676,12 @@ ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]tabl ##### RQ.SRS-006.RBAC.RowPolicy.Drop version: 1.0 -[ClickHouse] SHALL support removing one or more row policies using the `DROP ROW POLICY` statement. +[ClickHouse] SHALL support removing one or more row policies using the `CREATE ROW POLICY` statement. ##### RQ.SRS-006.RBAC.RowPolicy.Drop.IfExists version: 1.0 -[ClickHouse] SHALL support using the `IF EXISTS` clause in the `DROP ROW POLICY` statement +[ClickHouse] SHALL support using the `IF EXISTS` clause in the `CREATE ROW POLICY` statement to skip raising an exception when the row policy does not exist. If the `IF EXISTS` clause is not specified then an exception SHALL be raised if the row policy does not exist. @@ -2650,21 +2690,21 @@ raised if the row policy does not exist. version: 1.0 [ClickHouse] SHALL support removing row policy from one or more specified tables -using the `ON` clause in the `DROP ROW POLICY` statement. +using the `ON` clause in the `CREATE ROW POLICY` statement. ##### RQ.SRS-006.RBAC.RowPolicy.Drop.OnCluster version: 1.0 [ClickHouse] SHALL support removing row policy from specified cluster -using the `ON CLUSTER` clause in the `DROP ROW POLICY` statement. +using the `ON CLUSTER` clause in the `CREATE ROW POLICY` statement. ##### RQ.SRS-006.RBAC.RowPolicy.Drop.Syntax version: 1.0 -[ClickHouse] SHALL support the following syntax for the `DROP ROW POLICY` statement. +[ClickHouse] SHALL support the following syntax for the `CREATE ROW POLICY` statement. ``` sql -DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] +CREATE [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] ``` ##### RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy @@ -3124,14 +3164,14 @@ on tables created using the following engines * ReplicatedVersionedCollapsingMergeTree * ReplicatedGraphiteMergeTree -##### AlterColumn +##### Alter Column ###### RQ.SRS-006.RBAC.Privileges.AlterColumn version: 1.0 [ClickHouse] SHALL support controlling access to the **AlterColumn** privilege for a database or a specific table to one or more **users** or **roles**. -Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL +Any `ALTER TABLE ... ADD|CREATE|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL return an error, unless the user has the **alter column** privilege for the destination table either because of the explicit grant or through one of the roles assigned to the user. @@ -3153,7 +3193,7 @@ version: 1.0 [ClickHouse] SHALL support granting or revoking **alter column** privilege for one or more specified columns in a table to one or more **users** or **roles**. -Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL return an error, +Any `ALTER TABLE ... ADD|CREATE|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL return an error, unless the user has the **alter column** privilege for the destination column either because of the explicit grant or through one of the roles assigned to the user. @@ -3162,7 +3202,7 @@ version: 1.0 [ClickHouse] SHALL support granting or revoking **alter column** privilege on a specified cluster to one or more **users** or **roles**. -Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` +Any `ALTER TABLE ... ADD|CREATE|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL succeed only on nodes where the table exists and privilege was granted. ###### RQ.SRS-006.RBAC.Privileges.AlterColumn.TableEngines @@ -3186,14 +3226,14 @@ on tables created using the following engines * ReplicatedVersionedCollapsingMergeTree * ReplicatedGraphiteMergeTree -##### AlterIndex +##### Alter Index ###### RQ.SRS-006.RBAC.Privileges.AlterIndex version: 1.0 [ClickHouse] SHALL support controlling access to the **alter index** privilege for a database or a specific table to one or more **users** or **roles**. -Any `ALTER TABLE ... ORDER BY | ADD|DROP|MATERIALIZE|CLEAR INDEX` statements SHALL +Any `ALTER TABLE ... ORDER BY | ADD|CREATE|MATERIALIZE|CLEAR INDEX` statements SHALL return an error, unless the user has the **alter index** privilege for the destination table either because of the explicit grant or through one of the roles assigned to the user. @@ -3215,7 +3255,7 @@ version: 1.0 [ClickHouse] SHALL support granting or revoking **alter index** privilege on a specified cluster to one or more **users** or **roles**. -Any `ALTER TABLE ... ORDER BY | ADD|DROP|MATERIALIZE|CLEAR INDEX` +Any `ALTER TABLE ... ORDER BY | ADD|CREATE|MATERIALIZE|CLEAR INDEX` statements SHALL succeed only on nodes where the table exists and privilege was granted. ###### RQ.SRS-006.RBAC.Privileges.AlterIndex.TableEngines @@ -3239,14 +3279,14 @@ on tables created using the following engines * ReplicatedVersionedCollapsingMergeTree * ReplicatedGraphiteMergeTree -##### AlterConstraint +##### Alter Constraint ###### RQ.SRS-006.RBAC.Privileges.AlterConstraint version: 1.0 [ClickHouse] SHALL support controlling access to the **alter constraint** privilege for a database or a specific table to one or more **users** or **roles**. -Any `ALTER TABLE ... ADD|DROP CONSTRAINT` statements SHALL +Any `ALTER TABLE ... ADD|CREATE CONSTRAINT` statements SHALL return an error, unless the user has the **alter constraint** privilege for the destination table either because of the explicit grant or through one of the roles assigned to the user. @@ -3268,7 +3308,7 @@ version: 1.0 [ClickHouse] SHALL support granting or revoking **alter constraint** privilege on a specified cluster to one or more **users** or **roles**. -Any `ALTER TABLE ... ADD|DROP CONSTRAINT` +Any `ALTER TABLE ... ADD|CREATE CONSTRAINT` statements SHALL succeed only on nodes where the table exists and privilege was granted. ###### RQ.SRS-006.RBAC.Privileges.AlterConstraint.TableEngines @@ -3292,7 +3332,7 @@ on tables created using the following engines * ReplicatedVersionedCollapsingMergeTree * ReplicatedGraphiteMergeTree -##### AlterTTL +##### Alter TTL ###### RQ.SRS-006.RBAC.Privileges.AlterTTL version: 1.0 @@ -3332,7 +3372,7 @@ on tables created using the following engines * MergeTree -##### AlterSettings +##### Alter Settings ###### RQ.SRS-006.RBAC.Privileges.AlterSettings version: 1.0 @@ -3554,6 +3594,186 @@ on tables created using the following engines * ReplicatedVersionedCollapsingMergeTree * ReplicatedGraphiteMergeTree +##### Create Table + +###### RQ.SRS-006.RBAC.Privileges.CreateTable +version: 1.0 + +[ClickHouse] SHALL only successfully execute a `CREATE TABLE` command if and only if +the user has **create table** privilege either explicitly or through roles. + +If the stored query includes one or more source tables, the user must have **select** privilege +on all the source tables and **insert** for the table they're trying to create either explicitly or through a role. +For example, +```sql +CREATE TABLE table AS SELECT * FROM source_table +CREATE TABLE table AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression)) +CREATE TABLE table AS SELECT * FROM table0 JOIN table1 USING column +CREATE TABLE table AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2 +CREATE TABLE table AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression)) +CREATE TABLE table0 AS SELECT column FROM table1 UNION ALL SELECT column FROM table2 +``` + +###### RQ.SRS-006.RBAC.Privileges.CreateTable.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **create table** privilege to one or more **users** or **roles**. + +##### Create Database + +###### RQ.SRS-006.RBAC.Privileges.CreateDatabase +version: 1.0 + +[ClickHouse] SHALL successfully execute `CREATE DATABASE` statement if and only if the user has **create database** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.CreateDatabase.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **create database** privilege to one or more **users** or **roles**. + +##### Create Dictionary + +###### RQ.SRS-006.RBAC.Privileges.CreateDictionary +version: 1.0 + +[ClickHouse] SHALL successfully execute `CREATE DICTIONARY` statement if and only if the user has **create dictionary** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.CreateDictionary.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **create dictionary** privilege to one or more **users** or **roles**. + +##### Create Temporary Table + +###### RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable +version: 1.0 + +[ClickHouse] SHALL successfully execute `CREATE TEMPORARY TABLE` statement if and only if the user has **create temporary table** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **create temporary table** privilege to one or more **users** or **roles**. + +##### Attach Database + +###### RQ.SRS-006.RBAC.Privileges.AttachDatabase +version: 1.0 + +[ClickHouse] SHALL successfully execute `ATTACH DATABASE` statement if and only if the user has **create database** privilege on the source table, +either directly or through a role. + +##### Attach Dictionary + +###### RQ.SRS-006.RBAC.Privileges.AttachDictionary +version: 1.0 + +[ClickHouse] SHALL successfully execute `ATTACH DICTIONARY` statement if and only if the user has **create dictionary** privilege on the source table, +either directly or through a role. + +##### Attach Temporary Table + +###### RQ.SRS-006.RBAC.Privileges.AttachTemporaryTable +version: 1.0 + +[ClickHouse] SHALL successfully execute `ATTACH TEMPORARY TABLE` statement if and only if the user has **create temporary table** privilege on the source table, +either directly or through a role. + +##### Attach Table + +###### RQ.SRS-006.RBAC.Privileges.AttachTable +version: 1.0 + +[ClickHouse] SHALL successfully execute `ATTACH TABLE` statement if and only if the user has **create table** privilege on the source table, +either directly or through a role. + +##### Drop Table + +###### RQ.SRS-006.RBAC.Privileges.DropTable +version: 1.0 + +[ClickHouse] SHALL successfully execute `DROP TABLE` statement if and only if the user has **drop table** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.DropTable.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **drop table** privilege to one or more **users** or **roles**. + +##### Drop View + +###### RQ.SRS-006.RBAC.Privileges.DropView +version: 1.0 + +[ClickHouse] SHALL successfully execute `DROP VIEW` statement if and only if the user has **drop view** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.DropView.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **drop view** privilege to one or more **users** or **roles**. + +##### Drop Database + +###### RQ.SRS-006.RBAC.Privileges.DropDatabase +version: 1.0 + +[ClickHouse] SHALL successfully execute `DROP DATABASE` statement if and only if the user has **drop database** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.DropDatabase.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **drop database** privilege to one or more **users** or **roles**. + +##### Drop Dictionary + +###### RQ.SRS-006.RBAC.Privileges.DropDictionary +version: 1.0 + +[ClickHouse] SHALL successfully execute `DROP DICTIONARY` statement if and only if the user has **drop dictionary** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.DropDictionary.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **drop dictionary** privilege to one or more **users** or **roles**. + +##### Detach Table + +###### RQ.SRS-006.RBAC.Privileges.DetachTable +version: 1.0 + +[ClickHouse] SHALL successfully execute `DETACH TABLE` statement if and only if the user has **drop table** privilege on the source table, +either directly or through a role. + +##### Detach View + +###### RQ.SRS-006.RBAC.Privileges.DetachView +version: 1.0 + +[ClickHouse] SHALL successfully execute `DETACH VIEW` statement if and only if the user has **drop view** privilege on the source table, +either directly or through a role. + +##### Detach Database + +###### RQ.SRS-006.RBAC.Privileges.DetachDatabase +version: 1.0 + +[ClickHouse] SHALL successfully execute `DETACH DATABASE` statement if and only if the user has **drop database** privilege on the source table, +either directly or through a role. + +##### Detach Dictionary + +###### RQ.SRS-006.RBAC.Privileges.DetachDictionary +version: 1.0 + +[ClickHouse] SHALL successfully execute `DETACH DICTIONARY` statement if and only if the user has **drop dictionary** privilege on the source table, +either directly or through a role. + ##### Grant Option ###### RQ.SRS-006.RBAC.Privileges.GrantOption @@ -3576,6 +3796,15 @@ the user has that privilege with `GRANT OPTION`, either directly or through a ro * `ALTER INDEX` * `INSERT` * `SELECT` +* `CREATE TABLE` +* `CREATE VIEW` +* `CREATE DATABASE` +* `CREATE DICTIONARY` +* `CREATE TEMPORARY TABLE` +* `DROP TABLE` +* `DROP VIEW` +* `DROP DATABASE` +* `DROP DICTIONARY` ##### RQ.SRS-006.RBAC.Privileges.Delete version: 1.0 @@ -3639,14 +3868,14 @@ either because of the explicit grant or through one of the roles assigned to the ##### RQ.SRS-006.RBAC.RequiredPrivileges.Drop version: 1.0 -[ClickHouse] SHALL not allow any `DROP` statements +[ClickHouse] SHALL not allow any `CREATE` statements to be executed unless the user has the **drop** privilege for the destination database either because of the explicit grant or through one of the roles assigned to the user. ##### RQ.SRS-006.RBAC.RequiredPrivileges.Drop.Table version: 1.0 -[ClickHouse] SHALL not allow any `DROP TABLE` statements +[ClickHouse] SHALL not allow any `CREATE TABLE` statements to be executed unless the user has the **drop** privilege for the destination database or the table either because of the explicit grant or through one of the roles assigned to the user. diff --git a/tests/testflows/rbac/requirements/requirements.py b/tests/testflows/rbac/requirements/requirements.py index 5277b732066..cebeed4a04f 100755 --- a/tests/testflows/rbac/requirements/requirements.py +++ b/tests/testflows/rbac/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.201021.1163815. +# document by TestFlows v1.6.201102.1235648. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Specification @@ -411,32 +411,32 @@ SRS_006_ClickHouse_Role_Based_Access_Control = Specification( * 5.2.11.3.4 [RQ.SRS-006.RBAC.Privileges.Insert.Column](#rqsrs-006rbacprivilegesinsertcolumn) * 5.2.11.3.5 [RQ.SRS-006.RBAC.Privileges.Insert.Cluster](#rqsrs-006rbacprivilegesinsertcluster) * 5.2.11.3.6 [RQ.SRS-006.RBAC.Privileges.Insert.TableEngines](#rqsrs-006rbacprivilegesinserttableengines) - * 5.2.11.4 [AlterColumn](#altercolumn) + * 5.2.11.4 [Alter Column](#alter-column) * 5.2.11.4.1 [RQ.SRS-006.RBAC.Privileges.AlterColumn](#rqsrs-006rbacprivilegesaltercolumn) * 5.2.11.4.2 [RQ.SRS-006.RBAC.Privileges.AlterColumn.Grant](#rqsrs-006rbacprivilegesaltercolumngrant) * 5.2.11.4.3 [RQ.SRS-006.RBAC.Privileges.AlterColumn.Revoke](#rqsrs-006rbacprivilegesaltercolumnrevoke) * 5.2.11.4.4 [RQ.SRS-006.RBAC.Privileges.AlterColumn.Column](#rqsrs-006rbacprivilegesaltercolumncolumn) * 5.2.11.4.5 [RQ.SRS-006.RBAC.Privileges.AlterColumn.Cluster](#rqsrs-006rbacprivilegesaltercolumncluster) * 5.2.11.4.6 [RQ.SRS-006.RBAC.Privileges.AlterColumn.TableEngines](#rqsrs-006rbacprivilegesaltercolumntableengines) - * 5.2.11.5 [AlterIndex](#alterindex) + * 5.2.11.5 [Alter Index](#alter-index) * 5.2.11.5.1 [RQ.SRS-006.RBAC.Privileges.AlterIndex](#rqsrs-006rbacprivilegesalterindex) * 5.2.11.5.2 [RQ.SRS-006.RBAC.Privileges.AlterIndex.Grant](#rqsrs-006rbacprivilegesalterindexgrant) * 5.2.11.5.3 [RQ.SRS-006.RBAC.Privileges.AlterIndex.Revoke](#rqsrs-006rbacprivilegesalterindexrevoke) * 5.2.11.5.4 [RQ.SRS-006.RBAC.Privileges.AlterIndex.Cluster](#rqsrs-006rbacprivilegesalterindexcluster) * 5.2.11.5.5 [RQ.SRS-006.RBAC.Privileges.AlterIndex.TableEngines](#rqsrs-006rbacprivilegesalterindextableengines) - * 5.2.11.6 [AlterConstraint](#alterconstraint) + * 5.2.11.6 [Alter Constraint](#alter-constraint) * 5.2.11.6.1 [RQ.SRS-006.RBAC.Privileges.AlterConstraint](#rqsrs-006rbacprivilegesalterconstraint) * 5.2.11.6.2 [RQ.SRS-006.RBAC.Privileges.AlterConstraint.Grant](#rqsrs-006rbacprivilegesalterconstraintgrant) * 5.2.11.6.3 [RQ.SRS-006.RBAC.Privileges.AlterConstraint.Revoke](#rqsrs-006rbacprivilegesalterconstraintrevoke) * 5.2.11.6.4 [RQ.SRS-006.RBAC.Privileges.AlterConstraint.Cluster](#rqsrs-006rbacprivilegesalterconstraintcluster) * 5.2.11.6.5 [RQ.SRS-006.RBAC.Privileges.AlterConstraint.TableEngines](#rqsrs-006rbacprivilegesalterconstrainttableengines) - * 5.2.11.7 [AlterTTL](#alterttl) + * 5.2.11.7 [Alter TTL](#alter-ttl) * 5.2.11.7.1 [RQ.SRS-006.RBAC.Privileges.AlterTTL](#rqsrs-006rbacprivilegesalterttl) * 5.2.11.7.2 [RQ.SRS-006.RBAC.Privileges.AlterTTL.Grant](#rqsrs-006rbacprivilegesalterttlgrant) * 5.2.11.7.3 [RQ.SRS-006.RBAC.Privileges.AlterTTL.Revoke](#rqsrs-006rbacprivilegesalterttlrevoke) * 5.2.11.7.4 [RQ.SRS-006.RBAC.Privileges.AlterTTL.Cluster](#rqsrs-006rbacprivilegesalterttlcluster) * 5.2.11.7.5 [RQ.SRS-006.RBAC.Privileges.AlterTTL.TableEngines](#rqsrs-006rbacprivilegesalterttltableengines) - * 5.2.11.8 [AlterSettings](#altersettings) + * 5.2.11.8 [Alter Settings](#alter-settings) * 5.2.11.8.1 [RQ.SRS-006.RBAC.Privileges.AlterSettings](#rqsrs-006rbacprivilegesaltersettings) * 5.2.11.8.2 [RQ.SRS-006.RBAC.Privileges.AlterSettings.Grant](#rqsrs-006rbacprivilegesaltersettingsgrant) * 5.2.11.8.3 [RQ.SRS-006.RBAC.Privileges.AlterSettings.Revoke](#rqsrs-006rbacprivilegesaltersettingsrevoke) @@ -462,15 +462,55 @@ SRS_006_ClickHouse_Role_Based_Access_Control = Specification( * 5.2.11.13.1 [RQ.SRS-006.RBAC.Privileges.AlterMove](#rqsrs-006rbacprivilegesaltermove) * 5.2.11.13.2 [RQ.SRS-006.RBAC.Privileges.AlterMove.Access](#rqsrs-006rbacprivilegesaltermoveaccess) * 5.2.11.13.3 [RQ.SRS-006.RBAC.Privileges.AlterMove.TableEngines](#rqsrs-006rbacprivilegesaltermovetableengines) - * 5.2.11.14 [Grant Option](#grant-option) - * 5.2.11.14.1 [RQ.SRS-006.RBAC.Privileges.GrantOption](#rqsrs-006rbacprivilegesgrantoption) - * 5.2.11.15 [RQ.SRS-006.RBAC.Privileges.Delete](#rqsrs-006rbacprivilegesdelete) - * 5.2.11.16 [RQ.SRS-006.RBAC.Privileges.Alter](#rqsrs-006rbacprivilegesalter) - * 5.2.11.17 [RQ.SRS-006.RBAC.Privileges.Create](#rqsrs-006rbacprivilegescreate) - * 5.2.11.18 [RQ.SRS-006.RBAC.Privileges.Drop](#rqsrs-006rbacprivilegesdrop) - * 5.2.11.19 [RQ.SRS-006.RBAC.Privileges.All](#rqsrs-006rbacprivilegesall) - * 5.2.11.20 [RQ.SRS-006.RBAC.Privileges.All.GrantRevoke](#rqsrs-006rbacprivilegesallgrantrevoke) - * 5.2.11.21 [RQ.SRS-006.RBAC.Privileges.AdminOption](#rqsrs-006rbacprivilegesadminoption) + * 5.2.11.14 [Create Table](#create-table) + * 5.2.11.14.1 [RQ.SRS-006.RBAC.Privileges.CreateTable](#rqsrs-006rbaccreatetable) + * 5.2.11.14.2 [RQ.SRS-006.RBAC.Privileges.CreateTable.Access](#rqsrs-006rbacprivilegescreatetableaccess) + * 5.2.11.15 [Create Database](#create-database) + * 5.2.11.15.1 [RQ.SRS-006.RBAC.Privileges.CreateDatabase](#rqsrs-006rbacprivilegescreatedatabase) + * 5.2.11.15.2 [RQ.SRS-006.RBAC.Privileges.CreateDatabase.Access](#rqsrs-006rbacprivilegescreatedatabaseaccess) + * 5.2.11.16 [Create Dictionary](#create-dictionary) + * 5.2.11.16.1 [RQ.SRS-006.RBAC.Privileges.CreateDictionary](#rqsrs-006rbacprivilegescreatedictionary) + * 5.2.11.16.2 [RQ.SRS-006.RBAC.Privileges.CreateDictionary.Access](#rqsrs-006rbacprivilegescreatedictionaryaccess) + * 5.2.11.17 [Create Temporary Table](#create-temporary-table) + * 5.2.11.17.1 [RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable](#rqsrs-006rbacprivilegescreatetemporarytable) + * 5.2.11.17.2 [RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable.Access](#rqsrs-006rbacprivilegescreatetemporarytableaccess) + * 5.2.11.18 [Attach Database](#attach-database) + * 5.2.11.18.1 [RQ.SRS-006.RBAC.Privileges.AttachDatabase](#rqsrs-006rbacprivilegesattachdatabase) + * 5.2.11.19 [Attach Dictionary](#attach-dictionary) + * 5.2.11.19.1 [RQ.SRS-006.RBAC.Privileges.AttachDictionary](#rqsrs-006rbacprivilegesattachdictionary) + * 5.2.11.20 [Attach Temporary Table](#attach-temporary-table) + * 5.2.11.20.1 [RQ.SRS-006.RBAC.Privileges.AttachTemporaryTable](#rqsrs-006rbacprivilegesattachtemporarytable) + * 5.2.11.21 [Attach Table](#attach-table) + * 5.2.11.21.1 [RQ.SRS-006.RBAC.Privileges.AttachTable](#rqsrs-006rbacprivilegesattachtable) + * 5.2.11.22 [Drop Table](#drop-table) + * 5.2.11.22.1 [RQ.SRS-006.RBAC.Privileges.DropTable](#rqsrs-006rbacprivilegesdroptable) + * 5.2.11.22.2 [RQ.SRS-006.RBAC.Privileges.DropTable.Access](#rqsrs-006rbacprivilegesdroptableaccess) + * 5.2.11.23 [Drop View](#drop-view) + * 5.2.11.23.1 [RQ.SRS-006.RBAC.Privileges.DropView](#rqsrs-006rbacprivilegesdropview) + * 5.2.11.23.2 [RQ.SRS-006.RBAC.Privileges.DropView.Access](#rqsrs-006rbacprivilegesdropviewaccess) + * 5.2.11.24 [Drop Database](#drop-database) + * 5.2.11.24.1 [RQ.SRS-006.RBAC.Privileges.DropDatabase](#rqsrs-006rbacprivilegesdropdatabase) + * 5.2.11.24.2 [RQ.SRS-006.RBAC.Privileges.DropDatabase.Access](#rqsrs-006rbacprivilegesdropdatabaseaccess) + * 5.2.11.25 [Drop Dictionary](#drop-dictionary) + * 5.2.11.25.1 [RQ.SRS-006.RBAC.Privileges.DropDictionary](#rqsrs-006rbacprivilegesdropdictionary) + * 5.2.11.25.2 [RQ.SRS-006.RBAC.Privileges.DropDictionary.Access](#rqsrs-006rbacprivilegesdropdictionaryaccess) + * 5.2.11.26 [Detach Table](#detach-table) + * 5.2.11.26.1 [RQ.SRS-006.RBAC.Privileges.DetachTable](#rqsrs-006rbacprivilegesdetachtable) + * 5.2.11.27 [Detach View](#detach-view) + * 5.2.11.27.1 [RQ.SRS-006.RBAC.Privileges.DetachView](#rqsrs-006rbacprivilegesdetachview) + * 5.2.11.28 [Detach Database](#detach-database) + * 5.2.11.28.1 [RQ.SRS-006.RBAC.Privileges.DetachDatabase](#rqsrs-006rbacprivilegesdetachdatabase) + * 5.2.11.29 [Detach Dictionary](#detach-dictionary) + * 5.2.11.29.1 [RQ.SRS-006.RBAC.Privileges.DetachDictionary](#rqsrs-006rbacprivilegesdetachdictionary) + * 5.2.11.30 [Grant Option](#grant-option) + * 5.2.11.30.1 [RQ.SRS-006.RBAC.Privileges.GrantOption](#rqsrs-006rbacprivilegesgrantoption) + * 5.2.11.31 [RQ.SRS-006.RBAC.Privileges.Delete](#rqsrs-006rbacprivilegesdelete) + * 5.2.11.32 [RQ.SRS-006.RBAC.Privileges.Alter](#rqsrs-006rbacprivilegesalter) + * 5.2.11.33 [RQ.SRS-006.RBAC.Privileges.Create](#rqsrs-006rbacprivilegescreate) + * 5.2.11.34 [RQ.SRS-006.RBAC.Privileges.Drop](#rqsrs-006rbacprivilegesdrop) + * 5.2.11.35 [RQ.SRS-006.RBAC.Privileges.All](#rqsrs-006rbacprivilegesall) + * 5.2.11.36 [RQ.SRS-006.RBAC.Privileges.All.GrantRevoke](#rqsrs-006rbacprivilegesallgrantrevoke) + * 5.2.11.37 [RQ.SRS-006.RBAC.Privileges.AdminOption](#rqsrs-006rbacprivilegesadminoption) * 5.2.12 [Required Privileges](#required-privileges) * 5.2.12.1 [RQ.SRS-006.RBAC.RequiredPrivileges.Create](#rqsrs-006rbacrequiredprivilegescreate) * 5.2.12.2 [RQ.SRS-006.RBAC.RequiredPrivileges.Alter](#rqsrs-006rbacrequiredprivilegesalter) @@ -1052,7 +1092,7 @@ to some password as identification when altering user account using ##### RQ.SRS-006.RBAC.User.Alter.Host.AddDrop version: 1.0 -[ClickHouse] SHALL support altering user by adding and dropping access to hosts with the `ADD HOST` or the `DROP HOST`in the `ALTER USER` statement. +[ClickHouse] SHALL support altering user by adding and dropping access to hosts with the `ADD HOST` or the `CREATE HOST`in the `ALTER USER` statement. ##### RQ.SRS-006.RBAC.User.Alter.Host.Local version: 1.0 @@ -1147,7 +1187,7 @@ version: 1.0 ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] [RENAME TO new_name] [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] - [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [[ADD|CREATE] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] ``` @@ -1256,12 +1296,12 @@ SHOW CREATE USER [name | CURRENT_USER] ##### RQ.SRS-006.RBAC.User.Drop version: 1.0 -[ClickHouse] SHALL support removing a user account using `DROP USER` statement. +[ClickHouse] SHALL support removing a user account using `CREATE USER` statement. ##### RQ.SRS-006.RBAC.User.Drop.IfExists version: 1.0 -[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP USER` statement +[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE USER` statement to skip raising an exception if the user account does not exist. If the `IF EXISTS` clause is not specified then an exception SHALL be raised if a user does not exist. @@ -1269,16 +1309,16 @@ raised if a user does not exist. ##### RQ.SRS-006.RBAC.User.Drop.OnCluster version: 1.0 -[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP USER` statement +[ClickHouse] SHALL support using `ON CLUSTER` clause in the `CREATE USER` statement to specify the name of the cluster the user should be dropped from. ##### RQ.SRS-006.RBAC.User.Drop.Syntax version: 1.0 -[ClickHouse] SHALL support the following syntax for `DROP USER` statement +[ClickHouse] SHALL support the following syntax for `CREATE USER` statement ```sql -DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +CREATE USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` ##### RQ.SRS-006.RBAC.Role.Create @@ -1365,12 +1405,12 @@ ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] ##### RQ.SRS-006.RBAC.Role.Drop version: 1.0 -[ClickHouse] SHALL support removing one or more roles using `DROP ROLE` statement. +[ClickHouse] SHALL support removing one or more roles using `CREATE ROLE` statement. ##### RQ.SRS-006.RBAC.Role.Drop.IfExists version: 1.0 -[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP ROLE` statement +[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE ROLE` statement to skip raising an exception if the role does not exist. If the `IF EXISTS` clause is not specified then an exception SHALL be raised if a role does not exist. @@ -1378,15 +1418,15 @@ raised if a role does not exist. ##### RQ.SRS-006.RBAC.Role.Drop.Cluster version: 1.0 -[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP ROLE` statement to specify the cluster from which to drop the specified role. +[ClickHouse] SHALL support using `ON CLUSTER` clause in the `CREATE ROLE` statement to specify the cluster from which to drop the specified role. ##### RQ.SRS-006.RBAC.Role.Drop.Syntax version: 1.0 -[ClickHouse] SHALL support the following syntax for the `DROP ROLE` statement +[ClickHouse] SHALL support the following syntax for the `CREATE ROLE` statement ``` sql -DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +CREATE ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` ##### RQ.SRS-006.RBAC.Role.ShowCreate @@ -1444,7 +1484,7 @@ for a database or a table using the `GRANT CREATE` statement. version: 1.0 [ClickHouse] SHALL support granting the **drop** privilege to one or more users or roles -for a database or a table using the `GRANT DROP` statement. +for a database or a table using the `GRANT CREATE` statement. ##### RQ.SRS-006.RBAC.Grant.Privilege.Truncate version: 1.0 @@ -1570,7 +1610,7 @@ version: 1.0 [ClickHouse] SHALL support revoking ANY privilege to one or more users or roles for a database or a table using the `REVOKE some_privilege` statement. **some_privilege** refers to any Clickhouse defined privilege, whose hierarchy includes -SELECT, INSERT, ALTER, CREATE, DROP, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT, +SELECT, INSERT, ALTER, CREATE, CREATE, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT, SYSTEM, INTROSPECTION, SOURCES, dictGet and all of their sub-privileges. ##### RQ.SRS-006.RBAC.Revoke.Privilege.Select @@ -1601,7 +1641,7 @@ for a database or a table using the `REVOKE CREATE` statement. version: 1.0 [ClickHouse] SHALL support revoking the **drop** privilege to one or more users or roles -for a database or a table using the `REVOKE DROP` statement. +for a database or a table using the `REVOKE CREATE` statement. ##### RQ.SRS-006.RBAC.Revoke.Privilege.Truncate version: 1.0 @@ -1671,7 +1711,7 @@ version: 1.0 [ClickHouse] SHALL support revoking MULTIPLE **privileges** to one or more users or roles for a database or a table using the `REVOKE privilege1, privilege2...` statement. **privileges** refers to any set of Clickhouse defined privilege, whose hierarchy includes -SELECT, INSERT, ALTER, CREATE, DROP, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT, +SELECT, INSERT, ALTER, CREATE, CREATE, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT, SYSTEM, INTROSPECTION, SOURCES, dictGet and all of their sub-privileges. ##### RQ.SRS-006.RBAC.Revoke.Privilege.All @@ -2010,12 +2050,12 @@ ALTER SETTINGS PROFILE [IF EXISTS] name ##### RQ.SRS-006.RBAC.SettingsProfile.Drop version: 1.0 -[ClickHouse] SHALL support removing one or more settings profiles using the `DROP SETTINGS PROFILE` statement. +[ClickHouse] SHALL support removing one or more settings profiles using the `CREATE SETTINGS PROFILE` statement. ##### RQ.SRS-006.RBAC.SettingsProfile.Drop.IfExists version: 1.0 -[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP SETTINGS PROFILE` statement +[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE SETTINGS PROFILE` statement to skip raising an exception if the settings profile does not exist. If the `IF EXISTS` clause is not specified then an exception SHALL be raised if a settings profile does not exist. @@ -2024,15 +2064,15 @@ raised if a settings profile does not exist. version: 1.0 [ClickHouse] SHALL support dropping one or more settings profiles on specified cluster using -`ON CLUSTER` clause in the `DROP SETTINGS PROFILE` statement. +`ON CLUSTER` clause in the `CREATE SETTINGS PROFILE` statement. ##### RQ.SRS-006.RBAC.SettingsProfile.Drop.Syntax version: 1.0 -[ClickHouse] SHALL support the following syntax for the `DROP SETTINGS PROFILE` statement +[ClickHouse] SHALL support the following syntax for the `CREATE SETTINGS PROFILE` statement ``` sql -DROP SETTINGS PROFILE [IF EXISTS] name [,name,...] +CREATE SETTINGS PROFILE [IF EXISTS] name [,name,...] ``` ##### RQ.SRS-006.RBAC.SettingsProfile.ShowCreateSettingsProfile @@ -2367,12 +2407,12 @@ ALTER QUOTA [IF EXIST] name ##### RQ.SRS-006.RBAC.Quota.Drop version: 1.0 -[ClickHouse] SHALL support removing one or more quotas using the `DROP QUOTA` statement. +[ClickHouse] SHALL support removing one or more quotas using the `CREATE QUOTA` statement. ##### RQ.SRS-006.RBAC.Quota.Drop.IfExists version: 1.0 -[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP QUOTA` statement +[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE QUOTA` statement to skip raising an exception when the quota does not exist. If the `IF EXISTS` clause is not specified then an exception SHALL be raised if the quota does not exist. @@ -2380,16 +2420,16 @@ raised if the quota does not exist. ##### RQ.SRS-006.RBAC.Quota.Drop.Cluster version: 1.0 -[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP QUOTA` statement +[ClickHouse] SHALL support using `ON CLUSTER` clause in the `CREATE QUOTA` statement to indicate the cluster the quota to be dropped is located on. ##### RQ.SRS-006.RBAC.Quota.Drop.Syntax version: 1.0 -[ClickHouse] SHALL support the following syntax for the `DROP QUOTA` statement +[ClickHouse] SHALL support the following syntax for the `CREATE QUOTA` statement ``` sql -DROP QUOTA [IF EXISTS] name [,name...] +CREATE QUOTA [IF EXISTS] name [,name...] ``` ##### RQ.SRS-006.RBAC.Quota.ShowQuotas @@ -2661,12 +2701,12 @@ ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]tabl ##### RQ.SRS-006.RBAC.RowPolicy.Drop version: 1.0 -[ClickHouse] SHALL support removing one or more row policies using the `DROP ROW POLICY` statement. +[ClickHouse] SHALL support removing one or more row policies using the `CREATE ROW POLICY` statement. ##### RQ.SRS-006.RBAC.RowPolicy.Drop.IfExists version: 1.0 -[ClickHouse] SHALL support using the `IF EXISTS` clause in the `DROP ROW POLICY` statement +[ClickHouse] SHALL support using the `IF EXISTS` clause in the `CREATE ROW POLICY` statement to skip raising an exception when the row policy does not exist. If the `IF EXISTS` clause is not specified then an exception SHALL be raised if the row policy does not exist. @@ -2675,21 +2715,21 @@ raised if the row policy does not exist. version: 1.0 [ClickHouse] SHALL support removing row policy from one or more specified tables -using the `ON` clause in the `DROP ROW POLICY` statement. +using the `ON` clause in the `CREATE ROW POLICY` statement. ##### RQ.SRS-006.RBAC.RowPolicy.Drop.OnCluster version: 1.0 [ClickHouse] SHALL support removing row policy from specified cluster -using the `ON CLUSTER` clause in the `DROP ROW POLICY` statement. +using the `ON CLUSTER` clause in the `CREATE ROW POLICY` statement. ##### RQ.SRS-006.RBAC.RowPolicy.Drop.Syntax version: 1.0 -[ClickHouse] SHALL support the following syntax for the `DROP ROW POLICY` statement. +[ClickHouse] SHALL support the following syntax for the `CREATE ROW POLICY` statement. ``` sql -DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] +CREATE [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] ``` ##### RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy @@ -3149,14 +3189,14 @@ on tables created using the following engines * ReplicatedVersionedCollapsingMergeTree * ReplicatedGraphiteMergeTree -##### AlterColumn +##### Alter Column ###### RQ.SRS-006.RBAC.Privileges.AlterColumn version: 1.0 [ClickHouse] SHALL support controlling access to the **AlterColumn** privilege for a database or a specific table to one or more **users** or **roles**. -Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL +Any `ALTER TABLE ... ADD|CREATE|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL return an error, unless the user has the **alter column** privilege for the destination table either because of the explicit grant or through one of the roles assigned to the user. @@ -3178,7 +3218,7 @@ version: 1.0 [ClickHouse] SHALL support granting or revoking **alter column** privilege for one or more specified columns in a table to one or more **users** or **roles**. -Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL return an error, +Any `ALTER TABLE ... ADD|CREATE|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL return an error, unless the user has the **alter column** privilege for the destination column either because of the explicit grant or through one of the roles assigned to the user. @@ -3187,7 +3227,7 @@ version: 1.0 [ClickHouse] SHALL support granting or revoking **alter column** privilege on a specified cluster to one or more **users** or **roles**. -Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` +Any `ALTER TABLE ... ADD|CREATE|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL succeed only on nodes where the table exists and privilege was granted. ###### RQ.SRS-006.RBAC.Privileges.AlterColumn.TableEngines @@ -3211,14 +3251,14 @@ on tables created using the following engines * ReplicatedVersionedCollapsingMergeTree * ReplicatedGraphiteMergeTree -##### AlterIndex +##### Alter Index ###### RQ.SRS-006.RBAC.Privileges.AlterIndex version: 1.0 [ClickHouse] SHALL support controlling access to the **alter index** privilege for a database or a specific table to one or more **users** or **roles**. -Any `ALTER TABLE ... ORDER BY | ADD|DROP|MATERIALIZE|CLEAR INDEX` statements SHALL +Any `ALTER TABLE ... ORDER BY | ADD|CREATE|MATERIALIZE|CLEAR INDEX` statements SHALL return an error, unless the user has the **alter index** privilege for the destination table either because of the explicit grant or through one of the roles assigned to the user. @@ -3240,7 +3280,7 @@ version: 1.0 [ClickHouse] SHALL support granting or revoking **alter index** privilege on a specified cluster to one or more **users** or **roles**. -Any `ALTER TABLE ... ORDER BY | ADD|DROP|MATERIALIZE|CLEAR INDEX` +Any `ALTER TABLE ... ORDER BY | ADD|CREATE|MATERIALIZE|CLEAR INDEX` statements SHALL succeed only on nodes where the table exists and privilege was granted. ###### RQ.SRS-006.RBAC.Privileges.AlterIndex.TableEngines @@ -3264,14 +3304,14 @@ on tables created using the following engines * ReplicatedVersionedCollapsingMergeTree * ReplicatedGraphiteMergeTree -##### AlterConstraint +##### Alter Constraint ###### RQ.SRS-006.RBAC.Privileges.AlterConstraint version: 1.0 [ClickHouse] SHALL support controlling access to the **alter constraint** privilege for a database or a specific table to one or more **users** or **roles**. -Any `ALTER TABLE ... ADD|DROP CONSTRAINT` statements SHALL +Any `ALTER TABLE ... ADD|CREATE CONSTRAINT` statements SHALL return an error, unless the user has the **alter constraint** privilege for the destination table either because of the explicit grant or through one of the roles assigned to the user. @@ -3293,7 +3333,7 @@ version: 1.0 [ClickHouse] SHALL support granting or revoking **alter constraint** privilege on a specified cluster to one or more **users** or **roles**. -Any `ALTER TABLE ... ADD|DROP CONSTRAINT` +Any `ALTER TABLE ... ADD|CREATE CONSTRAINT` statements SHALL succeed only on nodes where the table exists and privilege was granted. ###### RQ.SRS-006.RBAC.Privileges.AlterConstraint.TableEngines @@ -3317,7 +3357,7 @@ on tables created using the following engines * ReplicatedVersionedCollapsingMergeTree * ReplicatedGraphiteMergeTree -##### AlterTTL +##### Alter TTL ###### RQ.SRS-006.RBAC.Privileges.AlterTTL version: 1.0 @@ -3357,7 +3397,7 @@ on tables created using the following engines * MergeTree -##### AlterSettings +##### Alter Settings ###### RQ.SRS-006.RBAC.Privileges.AlterSettings version: 1.0 @@ -3579,6 +3619,186 @@ on tables created using the following engines * ReplicatedVersionedCollapsingMergeTree * ReplicatedGraphiteMergeTree +##### Create Table + +###### RQ.SRS-006.RBAC.Privileges.CreateTable +version: 1.0 + +[ClickHouse] SHALL only successfully execute a `CREATE TABLE` command if and only if +the user has **create table** privilege either explicitly or through roles. + +If the stored query includes one or more source tables, the user must have **select** privilege +on all the source tables and **insert** for the table they're trying to create either explicitly or through a role. +For example, +```sql +CREATE TABLE table AS SELECT * FROM source_table +CREATE TABLE table AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression)) +CREATE TABLE table AS SELECT * FROM table0 JOIN table1 USING column +CREATE TABLE table AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2 +CREATE TABLE table AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression)) +CREATE TABLE table0 AS SELECT column FROM table1 UNION ALL SELECT column FROM table2 +``` + +###### RQ.SRS-006.RBAC.Privileges.CreateTable.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **create table** privilege to one or more **users** or **roles**. + +##### Create Database + +###### RQ.SRS-006.RBAC.Privileges.CreateDatabase +version: 1.0 + +[ClickHouse] SHALL successfully execute `CREATE DATABASE` statement if and only if the user has **create database** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.CreateDatabase.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **create database** privilege to one or more **users** or **roles**. + +##### Create Dictionary + +###### RQ.SRS-006.RBAC.Privileges.CreateDictionary +version: 1.0 + +[ClickHouse] SHALL successfully execute `CREATE DICTIONARY` statement if and only if the user has **create dictionary** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.CreateDictionary.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **create dictionary** privilege to one or more **users** or **roles**. + +##### Create Temporary Table + +###### RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable +version: 1.0 + +[ClickHouse] SHALL successfully execute `CREATE TEMPORARY TABLE` statement if and only if the user has **create temporary table** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **create temporary table** privilege to one or more **users** or **roles**. + +##### Attach Database + +###### RQ.SRS-006.RBAC.Privileges.AttachDatabase +version: 1.0 + +[ClickHouse] SHALL successfully execute `ATTACH DATABASE` statement if and only if the user has **create database** privilege on the source table, +either directly or through a role. + +##### Attach Dictionary + +###### RQ.SRS-006.RBAC.Privileges.AttachDictionary +version: 1.0 + +[ClickHouse] SHALL successfully execute `ATTACH DICTIONARY` statement if and only if the user has **create dictionary** privilege on the source table, +either directly or through a role. + +##### Attach Temporary Table + +###### RQ.SRS-006.RBAC.Privileges.AttachTemporaryTable +version: 1.0 + +[ClickHouse] SHALL successfully execute `ATTACH TEMPORARY TABLE` statement if and only if the user has **create temporary table** privilege on the source table, +either directly or through a role. + +##### Attach Table + +###### RQ.SRS-006.RBAC.Privileges.AttachTable +version: 1.0 + +[ClickHouse] SHALL successfully execute `ATTACH TABLE` statement if and only if the user has **create table** privilege on the source table, +either directly or through a role. + +##### Drop Table + +###### RQ.SRS-006.RBAC.Privileges.DropTable +version: 1.0 + +[ClickHouse] SHALL successfully execute `DROP TABLE` statement if and only if the user has **drop table** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.DropTable.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **drop table** privilege to one or more **users** or **roles**. + +##### Drop View + +###### RQ.SRS-006.RBAC.Privileges.DropView +version: 1.0 + +[ClickHouse] SHALL successfully execute `DROP VIEW` statement if and only if the user has **drop view** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.DropView.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **drop view** privilege to one or more **users** or **roles**. + +##### Drop Database + +###### RQ.SRS-006.RBAC.Privileges.DropDatabase +version: 1.0 + +[ClickHouse] SHALL successfully execute `DROP DATABASE` statement if and only if the user has **drop database** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.DropDatabase.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **drop database** privilege to one or more **users** or **roles**. + +##### Drop Dictionary + +###### RQ.SRS-006.RBAC.Privileges.DropDictionary +version: 1.0 + +[ClickHouse] SHALL successfully execute `DROP DICTIONARY` statement if and only if the user has **drop dictionary** privilege on the source table, +either directly or through a role. + +###### RQ.SRS-006.RBAC.Privileges.DropDictionary.Access +version: 1.0 + +[ClickHouse] SHALL support granting or revoking **drop dictionary** privilege to one or more **users** or **roles**. + +##### Detach Table + +###### RQ.SRS-006.RBAC.Privileges.DetachTable +version: 1.0 + +[ClickHouse] SHALL successfully execute `DETACH TABLE` statement if and only if the user has **drop table** privilege on the source table, +either directly or through a role. + +##### Detach View + +###### RQ.SRS-006.RBAC.Privileges.DetachView +version: 1.0 + +[ClickHouse] SHALL successfully execute `DETACH VIEW` statement if and only if the user has **drop view** privilege on the source table, +either directly or through a role. + +##### Detach Database + +###### RQ.SRS-006.RBAC.Privileges.DetachDatabase +version: 1.0 + +[ClickHouse] SHALL successfully execute `DETACH DATABASE` statement if and only if the user has **drop database** privilege on the source table, +either directly or through a role. + +##### Detach Dictionary + +###### RQ.SRS-006.RBAC.Privileges.DetachDictionary +version: 1.0 + +[ClickHouse] SHALL successfully execute `DETACH DICTIONARY` statement if and only if the user has **drop dictionary** privilege on the source table, +either directly or through a role. + ##### Grant Option ###### RQ.SRS-006.RBAC.Privileges.GrantOption @@ -3601,6 +3821,15 @@ the user has that privilege with `GRANT OPTION`, either directly or through a ro * `ALTER INDEX` * `INSERT` * `SELECT` +* `CREATE TABLE` +* `CREATE VIEW` +* `CREATE DATABASE` +* `CREATE DICTIONARY` +* `CREATE TEMPORARY TABLE` +* `DROP TABLE` +* `DROP VIEW` +* `DROP DATABASE` +* `DROP DICTIONARY` ##### RQ.SRS-006.RBAC.Privileges.Delete version: 1.0 @@ -3664,14 +3893,14 @@ either because of the explicit grant or through one of the roles assigned to the ##### RQ.SRS-006.RBAC.RequiredPrivileges.Drop version: 1.0 -[ClickHouse] SHALL not allow any `DROP` statements +[ClickHouse] SHALL not allow any `CREATE` statements to be executed unless the user has the **drop** privilege for the destination database either because of the explicit grant or through one of the roles assigned to the user. ##### RQ.SRS-006.RBAC.RequiredPrivileges.Drop.Table version: 1.0 -[ClickHouse] SHALL not allow any `DROP TABLE` statements +[ClickHouse] SHALL not allow any `CREATE TABLE` statements to be executed unless the user has the **drop** privilege for the destination database or the table either because of the explicit grant or through one of the roles assigned to the user. @@ -4856,7 +5085,7 @@ RQ_SRS_006_RBAC_User_Alter_Host_AddDrop = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support altering user by adding and dropping access to hosts with the `ADD HOST` or the `DROP HOST`in the `ALTER USER` statement.\n' + '[ClickHouse] SHALL support altering user by adding and dropping access to hosts with the `ADD HOST` or the `CREATE HOST`in the `ALTER USER` statement.\n' '\n' ), link=None) @@ -5071,7 +5300,7 @@ RQ_SRS_006_RBAC_User_Alter_Syntax = Requirement( 'ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name]\n' ' [RENAME TO new_name]\n' " [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]\n" - " [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]\n" + " [[ADD|CREATE] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]\n" ' [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]\n' " [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]\n" '```\n' @@ -5308,7 +5537,7 @@ RQ_SRS_006_RBAC_User_Drop = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support removing a user account using `DROP USER` statement.\n' + '[ClickHouse] SHALL support removing a user account using `CREATE USER` statement.\n' '\n' ), link=None) @@ -5321,7 +5550,7 @@ RQ_SRS_006_RBAC_User_Drop_IfExists = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP USER` statement\n' + '[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE USER` statement\n' 'to skip raising an exception if the user account does not exist.\n' 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' 'raised if a user does not exist.\n' @@ -5337,7 +5566,7 @@ RQ_SRS_006_RBAC_User_Drop_OnCluster = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP USER` statement\n' + '[ClickHouse] SHALL support using `ON CLUSTER` clause in the `CREATE USER` statement\n' 'to specify the name of the cluster the user should be dropped from.\n' '\n' ), @@ -5351,10 +5580,10 @@ RQ_SRS_006_RBAC_User_Drop_Syntax = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for `DROP USER` statement\n' + '[ClickHouse] SHALL support the following syntax for `CREATE USER` statement\n' '\n' '```sql\n' - 'DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name]\n' + 'CREATE USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name]\n' '```\n' '\n' ), @@ -5537,7 +5766,7 @@ RQ_SRS_006_RBAC_Role_Drop = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support removing one or more roles using `DROP ROLE` statement.\n' + '[ClickHouse] SHALL support removing one or more roles using `CREATE ROLE` statement.\n' '\n' ), link=None) @@ -5550,7 +5779,7 @@ RQ_SRS_006_RBAC_Role_Drop_IfExists = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP ROLE` statement\n' + '[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE ROLE` statement\n' 'to skip raising an exception if the role does not exist.\n' 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' 'raised if a role does not exist.\n' @@ -5566,7 +5795,7 @@ RQ_SRS_006_RBAC_Role_Drop_Cluster = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP ROLE` statement to specify the cluster from which to drop the specified role.\n' + '[ClickHouse] SHALL support using `ON CLUSTER` clause in the `CREATE ROLE` statement to specify the cluster from which to drop the specified role.\n' '\n' ), link=None) @@ -5579,10 +5808,10 @@ RQ_SRS_006_RBAC_Role_Drop_Syntax = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `DROP ROLE` statement\n' + '[ClickHouse] SHALL support the following syntax for the `CREATE ROLE` statement\n' '\n' '``` sql\n' - 'DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]\n' + 'CREATE ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]\n' '```\n' '\n' ), @@ -5712,7 +5941,7 @@ RQ_SRS_006_RBAC_Grant_Privilege_Drop = Requirement( uid=None, description=( '[ClickHouse] SHALL support granting the **drop** privilege to one or more users or roles\n' - 'for a database or a table using the `GRANT DROP` statement.\n' + 'for a database or a table using the `GRANT CREATE` statement.\n' '\n' ), link=None) @@ -5982,7 +6211,7 @@ RQ_SRS_006_RBAC_Revoke_Privilege_Any = Requirement( '[ClickHouse] SHALL support revoking ANY privilege to one or more users or roles\n' 'for a database or a table using the `REVOKE some_privilege` statement.\n' '**some_privilege** refers to any Clickhouse defined privilege, whose hierarchy includes\n' - 'SELECT, INSERT, ALTER, CREATE, DROP, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT,\n' + 'SELECT, INSERT, ALTER, CREATE, CREATE, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT,\n' 'SYSTEM, INTROSPECTION, SOURCES, dictGet and all of their sub-privileges.\n' '\n' ), @@ -6053,7 +6282,7 @@ RQ_SRS_006_RBAC_Revoke_Privilege_Drop = Requirement( uid=None, description=( '[ClickHouse] SHALL support revoking the **drop** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE DROP` statement.\n' + 'for a database or a table using the `REVOKE CREATE` statement.\n' '\n' ), link=None) @@ -6211,7 +6440,7 @@ RQ_SRS_006_RBAC_Revoke_Privilege_Multiple = Requirement( '[ClickHouse] SHALL support revoking MULTIPLE **privileges** to one or more users or roles\n' 'for a database or a table using the `REVOKE privilege1, privilege2...` statement.\n' '**privileges** refers to any set of Clickhouse defined privilege, whose hierarchy includes\n' - 'SELECT, INSERT, ALTER, CREATE, DROP, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT,\n' + 'SELECT, INSERT, ALTER, CREATE, CREATE, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT,\n' 'SYSTEM, INTROSPECTION, SOURCES, dictGet and all of their sub-privileges.\n' '\n' ), @@ -6918,7 +7147,7 @@ RQ_SRS_006_RBAC_SettingsProfile_Drop = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support removing one or more settings profiles using the `DROP SETTINGS PROFILE` statement.\n' + '[ClickHouse] SHALL support removing one or more settings profiles using the `CREATE SETTINGS PROFILE` statement.\n' '\n' ), link=None) @@ -6931,7 +7160,7 @@ RQ_SRS_006_RBAC_SettingsProfile_Drop_IfExists = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP SETTINGS PROFILE` statement\n' + '[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE SETTINGS PROFILE` statement\n' 'to skip raising an exception if the settings profile does not exist.\n' 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' 'raised if a settings profile does not exist.\n' @@ -6948,7 +7177,7 @@ RQ_SRS_006_RBAC_SettingsProfile_Drop_OnCluster = Requirement( uid=None, description=( '[ClickHouse] SHALL support dropping one or more settings profiles on specified cluster using\n' - '`ON CLUSTER` clause in the `DROP SETTINGS PROFILE` statement.\n' + '`ON CLUSTER` clause in the `CREATE SETTINGS PROFILE` statement.\n' '\n' ), link=None) @@ -6961,10 +7190,10 @@ RQ_SRS_006_RBAC_SettingsProfile_Drop_Syntax = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `DROP SETTINGS PROFILE` statement\n' + '[ClickHouse] SHALL support the following syntax for the `CREATE SETTINGS PROFILE` statement\n' '\n' '``` sql\n' - 'DROP SETTINGS PROFILE [IF EXISTS] name [,name,...]\n' + 'CREATE SETTINGS PROFILE [IF EXISTS] name [,name,...]\n' '```\n' '\n' ), @@ -7667,7 +7896,7 @@ RQ_SRS_006_RBAC_Quota_Drop = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support removing one or more quotas using the `DROP QUOTA` statement.\n' + '[ClickHouse] SHALL support removing one or more quotas using the `CREATE QUOTA` statement.\n' '\n' ), link=None) @@ -7680,7 +7909,7 @@ RQ_SRS_006_RBAC_Quota_Drop_IfExists = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP QUOTA` statement\n' + '[ClickHouse] SHALL support using `IF EXISTS` clause in the `CREATE QUOTA` statement\n' 'to skip raising an exception when the quota does not exist.\n' 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' 'raised if the quota does not exist.\n' @@ -7696,7 +7925,7 @@ RQ_SRS_006_RBAC_Quota_Drop_Cluster = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP QUOTA` statement\n' + '[ClickHouse] SHALL support using `ON CLUSTER` clause in the `CREATE QUOTA` statement\n' 'to indicate the cluster the quota to be dropped is located on.\n' '\n' ), @@ -7710,10 +7939,10 @@ RQ_SRS_006_RBAC_Quota_Drop_Syntax = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `DROP QUOTA` statement\n' + '[ClickHouse] SHALL support the following syntax for the `CREATE QUOTA` statement\n' '\n' '``` sql\n' - 'DROP QUOTA [IF EXISTS] name [,name...]\n' + 'CREATE QUOTA [IF EXISTS] name [,name...]\n' '```\n' '\n' ), @@ -8305,7 +8534,7 @@ RQ_SRS_006_RBAC_RowPolicy_Drop = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support removing one or more row policies using the `DROP ROW POLICY` statement.\n' + '[ClickHouse] SHALL support removing one or more row policies using the `CREATE ROW POLICY` statement.\n' '\n' ), link=None) @@ -8318,7 +8547,7 @@ RQ_SRS_006_RBAC_RowPolicy_Drop_IfExists = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support using the `IF EXISTS` clause in the `DROP ROW POLICY` statement\n' + '[ClickHouse] SHALL support using the `IF EXISTS` clause in the `CREATE ROW POLICY` statement\n' 'to skip raising an exception when the row policy does not exist.\n' 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' 'raised if the row policy does not exist.\n' @@ -8335,7 +8564,7 @@ RQ_SRS_006_RBAC_RowPolicy_Drop_On = Requirement( uid=None, description=( '[ClickHouse] SHALL support removing row policy from one or more specified tables\n' - 'using the `ON` clause in the `DROP ROW POLICY` statement.\n' + 'using the `ON` clause in the `CREATE ROW POLICY` statement.\n' '\n' ), link=None) @@ -8349,7 +8578,7 @@ RQ_SRS_006_RBAC_RowPolicy_Drop_OnCluster = Requirement( uid=None, description=( '[ClickHouse] SHALL support removing row policy from specified cluster\n' - 'using the `ON CLUSTER` clause in the `DROP ROW POLICY` statement.\n' + 'using the `ON CLUSTER` clause in the `CREATE ROW POLICY` statement.\n' '\n' ), link=None) @@ -8362,10 +8591,10 @@ RQ_SRS_006_RBAC_RowPolicy_Drop_Syntax = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `DROP ROW POLICY` statement.\n' + '[ClickHouse] SHALL support the following syntax for the `CREATE ROW POLICY` statement.\n' '\n' '``` sql\n' - 'DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name]\n' + 'CREATE [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name]\n' '```\n' '\n' ), @@ -9188,7 +9417,7 @@ RQ_SRS_006_RBAC_Privileges_AlterColumn = Requirement( description=( '[ClickHouse] SHALL support controlling access to the **AlterColumn** privilege\n' 'for a database or a specific table to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL\n' + 'Any `ALTER TABLE ... ADD|CREATE|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL\n' 'return an error, unless the user has the **alter column** privilege for\n' 'the destination table either because of the explicit grant or through one of\n' 'the roles assigned to the user.\n' @@ -9234,7 +9463,7 @@ RQ_SRS_006_RBAC_Privileges_AlterColumn_Column = Requirement( description=( '[ClickHouse] SHALL support granting or revoking **alter column** privilege\n' 'for one or more specified columns in a table to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL return an error,\n' + 'Any `ALTER TABLE ... ADD|CREATE|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL return an error,\n' ' unless the user has the **alter column** privilege for the destination column\n' 'either because of the explicit grant or through one of the roles assigned to the user.\n' '\n' @@ -9251,7 +9480,7 @@ RQ_SRS_006_RBAC_Privileges_AlterColumn_Cluster = Requirement( description=( '[ClickHouse] SHALL support granting or revoking **alter column** privilege\n' 'on a specified cluster to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN`\n' + 'Any `ALTER TABLE ... ADD|CREATE|CLEAR|COMMENT|MODIFY COLUMN`\n' 'statements SHALL succeed only on nodes where the table exists and privilege was granted.\n' '\n' ), @@ -9296,7 +9525,7 @@ RQ_SRS_006_RBAC_Privileges_AlterIndex = Requirement( description=( '[ClickHouse] SHALL support controlling access to the **alter index** privilege\n' 'for a database or a specific table to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ORDER BY | ADD|DROP|MATERIALIZE|CLEAR INDEX` statements SHALL\n' + 'Any `ALTER TABLE ... ORDER BY | ADD|CREATE|MATERIALIZE|CLEAR INDEX` statements SHALL\n' 'return an error, unless the user has the **alter index** privilege for\n' 'the destination table either because of the explicit grant or through one of\n' 'the roles assigned to the user.\n' @@ -9342,7 +9571,7 @@ RQ_SRS_006_RBAC_Privileges_AlterIndex_Cluster = Requirement( description=( '[ClickHouse] SHALL support granting or revoking **alter index** privilege\n' 'on a specified cluster to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ORDER BY | ADD|DROP|MATERIALIZE|CLEAR INDEX`\n' + 'Any `ALTER TABLE ... ORDER BY | ADD|CREATE|MATERIALIZE|CLEAR INDEX`\n' 'statements SHALL succeed only on nodes where the table exists and privilege was granted.\n' '\n' ), @@ -9387,7 +9616,7 @@ RQ_SRS_006_RBAC_Privileges_AlterConstraint = Requirement( description=( '[ClickHouse] SHALL support controlling access to the **alter constraint** privilege\n' 'for a database or a specific table to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ADD|DROP CONSTRAINT` statements SHALL\n' + 'Any `ALTER TABLE ... ADD|CREATE CONSTRAINT` statements SHALL\n' 'return an error, unless the user has the **alter constraint** privilege for\n' 'the destination table either because of the explicit grant or through one of\n' 'the roles assigned to the user.\n' @@ -9433,7 +9662,7 @@ RQ_SRS_006_RBAC_Privileges_AlterConstraint_Cluster = Requirement( description=( '[ClickHouse] SHALL support granting or revoking **alter constraint** privilege\n' 'on a specified cluster to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ADD|DROP CONSTRAINT`\n' + 'Any `ALTER TABLE ... ADD|CREATE CONSTRAINT`\n' 'statements SHALL succeed only on nodes where the table exists and privilege was granted.\n' '\n' ), @@ -9916,6 +10145,346 @@ RQ_SRS_006_RBAC_Privileges_AlterMove_TableEngines = Requirement( ), link=None) +RQ_SRS_006_RBAC_Privileges_CreateTable = Requirement( + name='RQ.SRS-006.RBAC.Privileges.CreateTable', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL only successfully execute a `CREATE TABLE` command if and only if\n' + 'the user has **create table** privilege either explicitly or through roles.\n' + '\n' + 'If the stored query includes one or more source tables, the user must have **select** privilege\n' + "on all the source tables and **insert** for the table they're trying to create either explicitly or through a role.\n" + 'For example,\n' + '```sql\n' + 'CREATE TABLE table AS SELECT * FROM source_table\n' + 'CREATE TABLE table AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n' + 'CREATE TABLE table AS SELECT * FROM table0 JOIN table1 USING column\n' + 'CREATE TABLE table AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n' + 'CREATE TABLE table AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n' + 'CREATE TABLE table0 AS SELECT column FROM table1 UNION ALL SELECT column FROM table2\n' + '```\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_CreateTable_Access = Requirement( + name='RQ.SRS-006.RBAC.Privileges.CreateTable.Access', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support granting or revoking **create table** privilege to one or more **users** or **roles**.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_CreateDatabase = Requirement( + name='RQ.SRS-006.RBAC.Privileges.CreateDatabase', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `CREATE DATABASE` statement if and only if the user has **create database** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_CreateDatabase_Access = Requirement( + name='RQ.SRS-006.RBAC.Privileges.CreateDatabase.Access', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support granting or revoking **create database** privilege to one or more **users** or **roles**.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_CreateDictionary = Requirement( + name='RQ.SRS-006.RBAC.Privileges.CreateDictionary', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `CREATE DICTIONARY` statement if and only if the user has **create dictionary** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_CreateDictionary_Access = Requirement( + name='RQ.SRS-006.RBAC.Privileges.CreateDictionary.Access', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support granting or revoking **create dictionary** privilege to one or more **users** or **roles**.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_CreateTemporaryTable = Requirement( + name='RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `CREATE TEMPORARY TABLE` statement if and only if the user has **create temporary table** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_CreateTemporaryTable_Access = Requirement( + name='RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable.Access', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support granting or revoking **create temporary table** privilege to one or more **users** or **roles**.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_AttachDatabase = Requirement( + name='RQ.SRS-006.RBAC.Privileges.AttachDatabase', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `ATTACH DATABASE` statement if and only if the user has **create database** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_AttachDictionary = Requirement( + name='RQ.SRS-006.RBAC.Privileges.AttachDictionary', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `ATTACH DICTIONARY` statement if and only if the user has **create dictionary** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_AttachTemporaryTable = Requirement( + name='RQ.SRS-006.RBAC.Privileges.AttachTemporaryTable', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `ATTACH TEMPORARY TABLE` statement if and only if the user has **create temporary table** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_AttachTable = Requirement( + name='RQ.SRS-006.RBAC.Privileges.AttachTable', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `ATTACH TABLE` statement if and only if the user has **create table** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DropTable = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DropTable', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `DROP TABLE` statement if and only if the user has **drop table** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DropTable_Access = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DropTable.Access', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support granting or revoking **drop table** privilege to one or more **users** or **roles**.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DropView = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DropView', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `DROP VIEW` statement if and only if the user has **drop view** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DropView_Access = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DropView.Access', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support granting or revoking **drop view** privilege to one or more **users** or **roles**.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DropDatabase = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DropDatabase', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `DROP DATABASE` statement if and only if the user has **drop database** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DropDatabase_Access = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DropDatabase.Access', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support granting or revoking **drop database** privilege to one or more **users** or **roles**.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DropDictionary = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DropDictionary', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `DROP DICTIONARY` statement if and only if the user has **drop dictionary** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DropDictionary_Access = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DropDictionary.Access', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support granting or revoking **drop dictionary** privilege to one or more **users** or **roles**.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DetachTable = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DetachTable', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `DETACH TABLE` statement if and only if the user has **drop table** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DetachView = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DetachView', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `DETACH VIEW` statement if and only if the user has **drop view** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DetachDatabase = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DetachDatabase', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `DETACH DATABASE` statement if and only if the user has **drop database** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + +RQ_SRS_006_RBAC_Privileges_DetachDictionary = Requirement( + name='RQ.SRS-006.RBAC.Privileges.DetachDictionary', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL successfully execute `DETACH DICTIONARY` statement if and only if the user has **drop dictionary** privilege on the source table,\n' + 'either directly or through a role.\n' + '\n' + ), + link=None) + RQ_SRS_006_RBAC_Privileges_GrantOption = Requirement( name='RQ.SRS-006.RBAC.Privileges.GrantOption', version='1.0', @@ -9941,6 +10510,15 @@ RQ_SRS_006_RBAC_Privileges_GrantOption = Requirement( '* `ALTER INDEX`\n' '* `INSERT`\n' '* `SELECT`\n' + '* `CREATE TABLE`\n' + '* `CREATE VIEW`\n' + '* `CREATE DATABASE`\n' + '* `CREATE DICTIONARY`\n' + '* `CREATE TEMPORARY TABLE`\n' + '* `DROP TABLE`\n' + '* `DROP VIEW`\n' + '* `DROP DATABASE`\n' + '* `DROP DICTIONARY`\n' '\n' ), link=None) @@ -10082,7 +10660,7 @@ RQ_SRS_006_RBAC_RequiredPrivileges_Drop = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL not allow any `DROP` statements\n' + '[ClickHouse] SHALL not allow any `CREATE` statements\n' 'to be executed unless the user has the **drop** privilege for the destination database\n' 'either because of the explicit grant or through one of the roles assigned to the user.\n' '\n' @@ -10097,7 +10675,7 @@ RQ_SRS_006_RBAC_RequiredPrivileges_Drop_Table = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL not allow any `DROP TABLE` statements\n' + '[ClickHouse] SHALL not allow any `CREATE TABLE` statements\n' 'to be executed unless the user has the **drop** privilege for the destination database or the table\n' 'either because of the explicit grant or through one of the roles assigned to the user.\n' '\n' diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_fetch.py b/tests/testflows/rbac/tests/privileges/alter/alter_fetch.py index e01ab244f38..ba2aac99cec 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_fetch.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_fetch.py @@ -24,13 +24,16 @@ def privilege_granted_directly_or_via_role(self, table_type, privilege, node=Non with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): with user(node, user_name): + with When(f"I run checks that {user_name} is only able to execute ALTER FETCH PARTITION with required privileges"): privilege_check(grant_target_name=user_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): with user(node, user_name), role(node, role_name): + with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") + with And(f"I run checks that {user_name} with {role_name} is only able to execute ALTER FETCH PARTITION with required privileges"): privilege_check(grant_target_name=role_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) @@ -42,6 +45,7 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No with Scenario("user without privilege", setup=instrument_clickhouse_server_log): table_name = f"merge_tree_{getuid()}" with table(node, table_name, table_type): + with When("I attempt to fetch a partition without privilege"): node.query(f"ALTER TABLE {table_name} FETCH PARTITION 1 FROM '/clickhouse/tables/{{shard}}/{table_name}'", settings = [("user", user_name)], exitcode=exitcode, message=message) @@ -49,8 +53,10 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No with Scenario("user with privilege", setup=instrument_clickhouse_server_log): table_name = f"merge_tree_{getuid()}" with table(node, table_name, table_type): + with When("I grant the fetch privilege"): node.query(f"GRANT {privilege} ON {table_name} TO {grant_target_name}") + with Then("I attempt to fetch a partition"): node.query(f"ALTER TABLE {table_name} FETCH PARTITION 1 FROM '/clickhouse/tables/{{shard}}/{table_name}'", settings = [("user", user_name)], exitcode=231, message="DB::Exception: No node") @@ -58,10 +64,13 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): table_name = f"merge_tree_{getuid()}" with table(node, table_name, table_type): + with When("I grant the fetch privilege"): node.query(f"GRANT {privilege} ON {table_name} TO {grant_target_name}") + with And("I revoke the fetch privilege"): node.query(f"REVOKE {privilege} ON {table_name} FROM {grant_target_name}") + with Then("I attempt to fetch a partition"): node.query(f"ALTER TABLE {table_name} FETCH PARTITION 1 FROM '/clickhouse/tables/{{shard}}/{table_name}'", settings = [("user", user_name)], exitcode=exitcode, message=message) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_move.py b/tests/testflows/rbac/tests/privileges/alter/alter_move.py index 57d7a7182a9..3972dc1ab05 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_move.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_move.py @@ -24,13 +24,16 @@ def privilege_granted_directly_or_via_role(self, table_type, privilege, node=Non with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): with user(node, user_name): + with When(f"I run checks that {user_name} is only able to execute ALTER MOVE PARTITION with required privileges"): privilege_check(grant_target_name=user_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): with user(node, user_name), role(node, role_name): + with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") + with And(f"I run checks that {user_name} with {role_name} is only able to execute ALTER MOVE PARTITION with required privileges"): privilege_check(grant_target_name=role_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) @@ -44,6 +47,7 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No target_table_name = f"target_merge_tree_{getuid()}" with table(node, f"{source_table_name},{target_table_name}", table_type): + with When("I attempt to move partition without privilege"): node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {target_table_name}", settings = [("user", user_name)], exitcode=exitcode, message=message) @@ -53,8 +57,10 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No target_table_name = f"target_merge_tree_{getuid()}" with table(node, f"{source_table_name},{target_table_name}", table_type): + with When(f"I grant SELECT and ALTER DELETE privileges on {source_table_name} to {grant_target_name}"): node.query(f"GRANT SELECT, ALTER DELETE ON {source_table_name} TO {grant_target_name}") + with And(f"I grant INSERT on {target_table_name} to {grant_target_name}"): node.query(f"GRANT INSERT ON {target_table_name} TO {grant_target_name}") @@ -67,8 +73,10 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No target_table_name = f"target_merge_tree_{getuid()}" with table(node, f"{source_table_name},{target_table_name}", table_type): + with When(f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}"): node.query(f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}") + with And(f"I grant INSERT on {target_table_name} to {grant_target_name}"): node.query(f"GRANT INSERT ON {target_table_name} TO {grant_target_name}") @@ -80,8 +88,10 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No target_table_name = f"target_merge_tree_{getuid()}" with table(node, f"{source_table_name},{target_table_name}", table_type): + with When(f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}"): node.query(f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}") + with And(f"I grant INSERT on {target_table_name} to {grant_target_name}"): node.query(f"GRANT INSERT ON {target_table_name} TO {grant_target_name}") @@ -98,16 +108,23 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No mat_view_source_table_name = f"mat_view_source_merge_tree_{getuid()}" with table(node, f"{source_table_name},{mat_view_source_table_name}", table_type): - with Given("I have a materialized view"): - node.query(f"CREATE MATERIALIZED VIEW {mat_view_name} ENGINE = {table_type} PARTITION BY y ORDER BY d AS SELECT * FROM {mat_view_source_table_name}") - with When(f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}"): - node.query(f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}") - with And(f"I grant INSERT on {mat_view_source_table_name} to {grant_target_name}"): - node.query(f"GRANT INSERT ON {mat_view_source_table_name} TO {grant_target_name}") + try: + with Given("I have a materialized view"): + node.query(f"CREATE MATERIALIZED VIEW {mat_view_name} ENGINE = {table_type} PARTITION BY y ORDER BY d AS SELECT * FROM {mat_view_source_table_name}") - with Then("I attempt to move partitions with ALTER MOVE privilege"): - node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {mat_view_source_table_name}", settings = [("user", user_name)]) + with When(f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}"): + node.query(f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}") + + with And(f"I grant INSERT on {mat_view_source_table_name} to {grant_target_name}"): + node.query(f"GRANT INSERT ON {mat_view_source_table_name} TO {grant_target_name}") + + with Then("I attempt to move partitions with ALTER MOVE privilege"): + node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {mat_view_source_table_name}", settings = [("user", user_name)]) + + finally: + with Finally("I drop the materialized view"): + node.query(f"DROP VIEW IF EXISTS {mat_view_name}") with Scenario("move partition to implicit target table of a materialized view", setup=instrument_clickhouse_server_log): source_table_name = f"source_merge_tree_{getuid()}" @@ -116,16 +133,23 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No implicit_table_name = f"\\\".inner.{mat_view_name}\\\"" with table(node, f"{source_table_name},{mat_view_source_table_name}", table_type): - with Given("I have a materialized view"): - node.query(f"CREATE MATERIALIZED VIEW {mat_view_name} ENGINE = {table_type} PARTITION BY y ORDER BY d AS SELECT * FROM {mat_view_source_table_name}") - with When(f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}"): - node.query(f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}") - with And(f"I grant INSERT on {implicit_table_name} to {grant_target_name}"): - node.query(f"GRANT INSERT ON {implicit_table_name} TO {grant_target_name}") + try: + with Given("I have a materialized view"): + node.query(f"CREATE MATERIALIZED VIEW {mat_view_name} ENGINE = {table_type} PARTITION BY y ORDER BY d AS SELECT * FROM {mat_view_source_table_name}") - with Then("I attempt to move partitions with ALTER MOVE privilege"): - node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {implicit_table_name}", settings = [("user", user_name)]) + with When(f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}"): + node.query(f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}") + + with And(f"I grant INSERT on {implicit_table_name} to {grant_target_name}"): + node.query(f"GRANT INSERT ON {implicit_table_name} TO {grant_target_name}") + + with Then("I attempt to move partitions with ALTER MOVE privilege"): + node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {implicit_table_name}", settings = [("user", user_name)]) + + finally: + with Finally("I drop the materialized view"): + node.query(f"DROP VIEW IF EXISTS {mat_view_name}") @TestFeature @Requirements( diff --git a/tests/testflows/rbac/tests/privileges/attach/__init__.py b/tests/testflows/rbac/tests/privileges/attach/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/rbac/tests/privileges/attach/attach_database.py b/tests/testflows/rbac/tests/privileges/attach/attach_database.py new file mode 100644 index 00000000000..e7d85863cc5 --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/attach/attach_database.py @@ -0,0 +1,99 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateDatabase_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute ATTACH DATABASE when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute CREATE DATABASE with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE DATABASE with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with When("I attempt to attach a database without privilege"): + node.query(f"ATTACH DATABASE {db_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with When("I grant create database privilege"): + node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + + with Then("I attempt to attach aa database"): + node.query(f"ATTACH DATABASE {db_name}", settings = [("user", user_name)], + exitcode=80, message="DB::Exception: Received from localhost:9000. DB::Exception: Database engine must be specified for ATTACH DATABASE query") + + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with When("I grant the create database privilege"): + node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + + with And("I revoke the create database privilege"): + node.query(f"REVOKE CREATE DATABASE ON {db_name}.* FROM {grant_target_name}") + + with Then("I attempt to attach a database"): + node.query(f"ATTACH DATABASE {db_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_AttachDatabase("1.0"), +) +@Name("attach database") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of ATTACH DATABASE. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/attach/attach_dictionary.py b/tests/testflows/rbac/tests/privileges/attach/attach_dictionary.py new file mode 100644 index 00000000000..b635b1c42c6 --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/attach/attach_dictionary.py @@ -0,0 +1,99 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateDictionary_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute ATTACH DICTIONARY when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute CREATE DICTIONARY with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE DICTIONARY with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with When("I attempt to attach a dictionary without privilege"): + node.query(f"ATTACH DICTIONARY {dict_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with When("I grant create dictionary privilege"): + node.query(f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}") + + with Then("I attempt to attach a dictionary"): + node.query(f"ATTACH DICTIONARY {dict_name}", settings = [("user", user_name)], + exitcode=231, message=f"DB::Exception: Dictionary `{dict_name}` doesn't exist.") + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with When("I grant the create dictionary privilege"): + node.query(f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}") + + with And("I revoke the create dictionary privilege"): + node.query(f"REVOKE CREATE DICTIONARY ON {dict_name} FROM {grant_target_name}") + + with Then("I attempt to attach a dictionary"): + node.query(f"ATTACH DICTIONARY {dict_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_AttachDictionary("1.0"), +) +@Name("attach dictionary") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of ATTACH DICTIONARY. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/attach/attach_table.py b/tests/testflows/rbac/tests/privileges/attach/attach_table.py new file mode 100644 index 00000000000..e4322e93b92 --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/attach/attach_table.py @@ -0,0 +1,99 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateTable_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute ATTACH TABLE when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute CREATE TABLE with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE TABLE with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + table_name = f"table_{getuid()}" + + try: + with When("I attempt to attach a table without privilege"): + node.query(f"ATTACH TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + table_name = f"table_{getuid()}" + + try: + with When("I grant create table privilege"): + node.query(f"GRANT CREATE TABLE ON *.* TO {grant_target_name}") + + with Then("I attempt to attach a table"): + node.query(f"ATTACH TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", user_name)], + exitcode=80, message="DB::Exception: UUID must be specified") + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + table_name = f"table_{getuid()}" + + try: + with When("I grant the create table privilege"): + node.query(f"GRANT CREATE TABLE ON *.* TO {grant_target_name}") + + with And("I revoke the create table privilege"): + node.query(f"REVOKE CREATE TABLE ON *.* FROM {grant_target_name}") + + with Then("I attempt to attach a table"): + node.query(f"ATTACH TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_AttachTable("1.0"), +) +@Name("attach table") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of ATTACH TABLE. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/attach/attach_temp_table.py b/tests/testflows/rbac/tests/privileges/attach/attach_temp_table.py new file mode 100644 index 00000000000..b70c97d43f0 --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/attach/attach_temp_table.py @@ -0,0 +1,97 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateTemporaryTable_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute ATTACH TEMPORARY TABLE when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute CREATE TEMPORARY TABLE with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE TEMPORARY TABLE with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + temp_table_name = f"temp_table_{getuid()}" + + try: + with When("I attempt to attach a temporary table without privilege"): + node.query(f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the temporary table"): + node.query(f"DROP TEMPORARY TABLE IF EXISTS {temp_table_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + temp_table_name = f"temp_table_{getuid()}" + + try: + with When("I grant create temporary table privilege"): + node.query(f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}") + with Then("I attempt to attach a temporary table"): + node.query(f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)]) + + finally: + with Finally("I drop the temporary table"): + node.query(f"DROP TEMPORARY TABLE IF EXISTS {temp_table_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + temp_table_name = f"temp_table_{getuid()}" + + try: + with When("I grant the create temporary table privilege"): + node.query(f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}") + + with And("I revoke the create temporary table privilege"): + node.query(f"REVOKE CREATE TEMPORARY TABLE ON *.* FROM {grant_target_name}") + + with Then("I attempt to attach a temporary table"): + node.query(f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the temporary table"): + node.query(f"DROP TEMPORARY TABLE IF EXISTS {temp_table_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_AttachTemporaryTable("1.0"), +) +@Name("attach temporary table") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of ATTACH TEMPORARY TABLE. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/create/__init__.py b/tests/testflows/rbac/tests/privileges/create/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/rbac/tests/privileges/create/create_database.py b/tests/testflows/rbac/tests/privileges/create/create_database.py new file mode 100644 index 00000000000..d3411240712 --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/create/create_database.py @@ -0,0 +1,98 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateDatabase_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute CREATE DATABASE when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute CREATE DATABASE with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE DATABASE with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with When("I attempt to create a database without privilege"): + node.query(f"CREATE DATABASE {db_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with When("I grant create database privilege"): + node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + + with Then("I attempt to create a database"): + node.query(f"CREATE DATABASE {db_name}", settings = [("user", user_name)]) + + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with When("I grant the create database privilege"): + node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + + with And("I revoke the create database privilege"): + node.query(f"REVOKE CREATE DATABASE ON {db_name}.* FROM {grant_target_name}") + + with Then("I attempt to create a database"): + node.query(f"CREATE DATABASE {db_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateDatabase("1.0"), +) +@Name("create database") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of CREATE DATABASE. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/create/create_dictionary.py b/tests/testflows/rbac/tests/privileges/create/create_dictionary.py new file mode 100644 index 00000000000..6eb64b8e5ca --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/create/create_dictionary.py @@ -0,0 +1,98 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateDictionary_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute CREATE DICTIONARY when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute CREATE DICTIONARY with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE DICTIONARY with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with When("I attempt to create a dictionary without privilege"): + node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with When("I grant create dictionary privilege"): + node.query(f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}") + + with Then("I attempt to create a dictionary"): + node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", settings = [("user", user_name)]) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with When("I grant the create dictionary privilege"): + node.query(f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}") + + with And("I revoke the create dictionary privilege"): + node.query(f"REVOKE CREATE DICTIONARY ON {dict_name} FROM {grant_target_name}") + + with Then("I attempt to create a dictionary"): + node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateDictionary("1.0"), +) +@Name("create dictionary") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of CREATE DICTIONARY. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/create/create_table.py b/tests/testflows/rbac/tests/privileges/create/create_table.py new file mode 100644 index 00000000000..ddb94e2acad --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/create/create_table.py @@ -0,0 +1,731 @@ +from testflows.core import * +from testflows.asserts import error + +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestScenario +def create_without_create_table_privilege(self, node=None): + """Check that user is unable to create a table without CREATE TABLE privilege. + """ + user_name = f"user_{getuid()}" + table_name = f"table_{getuid()}" + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + if node is None: + node = self.context.node + + with user(node, f"{user_name}"): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with When("I try to create a table without CREATE TABLE privilege as the user"): + node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", f"{user_name}")], + exitcode=exitcode, message=message) + +@TestScenario +def create_with_create_table_privilege_granted_directly_or_via_role(self, node=None): + """Check that user is able to create a table with CREATE TABLE privilege, either granted directly or through a role. + """ + user_name = f"user_{getuid()}" + role_name = f"role_{getuid()}" + + if node is None: + node = self.context.node + + with user(node, f"{user_name}"): + + Scenario(test=create_with_create_table_privilege, + name="create with create table privilege granted directly")(grant_target_name=user_name, user_name=user_name) + + with user(node, f"{user_name}"), role(node, f"{role_name}"): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + Scenario(test=create_with_create_table_privilege, + name="create with create table privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + +@TestOutline +def create_with_create_table_privilege(self, grant_target_name, user_name, node=None): + """Check that user is able to create a table with the granted privileges. + """ + table_name = f"table_{getuid()}" + + if node is None: + node = self.context.node + try: + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with When("I grant the CREATE TABLE privilege"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") + + with Then("I try to create a table without privilege as the user"): + node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", f"{user_name}")]) + + finally: + with Then("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestScenario +def create_with_revoked_create_table_privilege_revoked_directly_or_from_role(self, node=None): + """Check that user is unable to create table after the CREATE TABLE privilege is revoked, either directly or from a role. + """ + user_name = f"user_{getuid()}" + role_name = f"role_{getuid()}" + + if node is None: + node = self.context.node + + with user(node, f"{user_name}"): + + Scenario(test=create_with_revoked_create_table_privilege, + name="create with create table privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + + with user(node, f"{user_name}"), role(node, f"{role_name}"): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + Scenario(test=create_with_revoked_create_table_privilege, + name="create with create table privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + +@TestOutline +def create_with_revoked_create_table_privilege(self, grant_target_name, user_name, node=None): + """Revoke CREATE TABLE privilege and check the user is unable to create a table. + """ + table_name = f"table_{getuid()}" + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + if node is None: + node = self.context.node + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with When("I grant CREATE TABLE privilege"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") + + with And("I revoke CREATE TABLE privilege"): + node.query(f"REVOKE CREATE TABLE ON {table_name} FROM {grant_target_name}") + + with Then("I try to create a table on the table as the user"): + node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", f"{user_name}")], + exitcode=exitcode, message=message) + +@TestScenario +def create_without_source_table_privilege(self, node=None): + """Check that user is unable to create a table without select + privilege on the source table. + """ + user_name = f"user_{getuid()}" + table_name = f"table_{getuid()}" + source_table_name = f"source_table_{getuid()}" + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + if node is None: + node = self.context.node + + with table(node, f"{source_table_name}"): + with user(node, f"{user_name}"): + + with When("I grant CREATE TABLE privilege to a user"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {user_name}") + + with And("I grant INSERT privilege"): + node.query(f"GRANT INSERT ON {table_name} TO {user_name}") + + with Then("I try to create a table without select privilege on the table"): + node.query(f"CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {source_table_name}", settings = [("user", f"{user_name}")], + exitcode=exitcode, message=message) + +@TestScenario +def create_without_insert_privilege(self, node=None): + """Check that user is unable to create a table without insert + privilege on that table. + """ + user_name = f"user_{getuid()}" + table_name = f"table_{getuid()}" + source_table_name = f"source_table_{getuid()}" + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + if node is None: + node = self.context.node + + with table(node, f"{source_table_name}"): + with user(node, f"{user_name}"): + + with When("I grant CREATE TABLE privilege to a user"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {user_name}") + + with And("I grant SELECT privilege"): + node.query(f"GRANT SELECT ON {source_table_name} TO {user_name}") + + with Then("I try to create a table without select privilege on the table"): + node.query(f"CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {source_table_name}", settings = [("user", f"{user_name}")], + exitcode=exitcode, message=message) + +@TestScenario +def create_with_source_table_privilege_granted_directly_or_via_role(self, node=None): + """Check that a user is able to create a table if and only if the user has create table privilege and + select privilege on the source table, either granted directly or through a role. + """ + user_name = f"user_{getuid()}" + role_name = f"role_{getuid()}" + + if node is None: + node = self.context.node + + with user(node, f"{user_name}"): + + Scenario(test=create_with_source_table_privilege, + name="create with create table and select privilege granted directly")(grant_target_name=user_name, user_name=user_name) + + with user(node, f"{user_name}"), role(node, f"{role_name}"): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + Scenario(test=create_with_source_table_privilege, + name="create with create table and select privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + +@TestOutline +def create_with_source_table_privilege(self, user_name, grant_target_name, node=None): + """Check that user is unable to create a table without SELECT privilege on the source table. + """ + table_name = f"table_{getuid()}" + source_table_name = f"source_table_{getuid()}" + + if node is None: + node = self.context.node + + with table(node, f"{source_table_name}"): + + try: + with When("I grant CREATE table privilege"): + node.query(f"GRANT CREATE table ON {table_name} TO {grant_target_name}") + + with And("I grant INSERT privilege"): + node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") + + with And("I grant SELECT privilege"): + node.query(f"GRANT SELECT ON {source_table_name} TO {grant_target_name}") + + with And("I try to create a table on the table as the user"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + node.query(f"CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {source_table_name}", settings = [("user", f"{user_name}")]) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestScenario +def create_with_subquery_privilege_granted_directly_or_via_role(self, node=None): + """Check that user is able to create a table where the stored query has two subqueries + if and only if the user has SELECT privilege on all of the tables, + either granted directly or through a role. + """ + user_name = f"user_{getuid()}" + role_name = f"role_{getuid()}" + + if node is None: + node = self.context.node + + with user(node, f"{user_name}"): + + Scenario(test=create_with_subquery, + name="create with subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + + with user(node, f"{user_name}"), role(node, f"{role_name}"): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + Scenario(test=create_with_subquery, + name="create with subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + +@TestOutline +def create_with_subquery(self, user_name, grant_target_name, node=None): + """Grant select and create table privileges and check that user is able to create a table + if and only if they have all necessary privileges. + """ + table_name = f"table_{getuid()}" + table0_name = f"table0_{getuid()}" + table1_name = f"table1_{getuid()}" + table2_name = f"table2_{getuid()}" + + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + create_table_query = "CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {table0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table2_name} WHERE y<2))" + + if node is None: + node = self.context.node + + with table(node, f"{table0_name},{table1_name},{table2_name}"): + + try: + with When("I grant CREATE TABLE privilege"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") + + with And("I grant INSERT privilege"): + node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") + + with Then("I attempt to CREATE TABLE as the user with create privilege"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], + exitcode=exitcode, message=message) + + for permutation in permutations(table_count=3): + with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name) as tables_granted: + + with When(f"permutation={permutation}, tables granted = {tables_granted}"): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Then("I attempt to create a table as the user"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], + exitcode=exitcode, message=message) + + with When("I grant select on all tables"): + with grant_select_on_table(node, max(permutations(table_count=3))+1, grant_target_name, table0_name, table1_name, table2_name): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Then("I attempt to create a table as the user"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")]) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestScenario +def create_with_join_query_privilege_granted_directly_or_via_role(self, node=None): + """Check that user is able to create a table where the stored query includes a `JOIN` statement + if and only if the user has SELECT privilege on all of the tables, + either granted directly or through a role. + """ + user_name = f"user_{getuid()}" + role_name = f"role_{getuid()}" + + if node is None: + node = self.context.node + + with user(node, f"{user_name}"): + + Scenario(test=create_with_join_query, + name="create with join query, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + + with user(node, f"{user_name}"), role(node, f"{role_name}"): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + Scenario(test=create_with_join_query, + name="create with join query, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + +@TestOutline +def create_with_join_query(self, grant_target_name, user_name, node=None): + """Grant select and create table privileges and check that user is able to create a table + if and only if they have all necessary privileges. + """ + table_name = f"table_{getuid()}" + table0_name = f"table0_{getuid()}" + table1_name = f"table1_{getuid()}" + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + create_table_query = "CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {table0_name} JOIN {table1_name} USING d" + + if node is None: + node = self.context.node + + with table(node, f"{table0_name},{table1_name}"): + + try: + with When("I grant CREATE TABLE privilege"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") + + with And("I grant INSERT privilege"): + node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") + + with Then("I attempt to create table as the user"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], + exitcode=exitcode, message=message) + + for permutation in permutations(table_count=2): + with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: + + with When(f"permutation={permutation}, tables granted = {tables_granted}"): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Then("I attempt to create a table as the user"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], + exitcode=exitcode, message=message) + + with When("I grant select on all tables"): + with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Then("I attempt to create a table as the user"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + + finally: + with Then("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestScenario +def create_with_union_query_privilege_granted_directly_or_via_role(self, node=None): + """Check that user is able to create a table where the stored query includes a `UNION ALL` statement + if and only if the user has SELECT privilege on all of the tables, + either granted directly or through a role. + """ + user_name = f"user_{getuid()}" + role_name = f"role_{getuid()}" + + if node is None: + node = self.context.node + + with user(node, f"{user_name}"): + + Scenario(test=create_with_union_query, + name="create with union query, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + + with user(node, f"{user_name}"), role(node, f"{role_name}"): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + Scenario(test=create_with_union_query, + name="create with union query, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + +@TestOutline +def create_with_union_query(self, grant_target_name, user_name, node=None): + """Grant select and create table privileges and check that user is able to create a table + if and only if they have all necessary privileges. + """ + table_name = f"table_{getuid()}" + table0_name = f"table0_{getuid()}" + table1_name = f"table1_{getuid()}" + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + create_table_query = "CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {table0_name} UNION ALL SELECT * FROM {table1_name}" + + if node is None: + node = self.context.node + + with table(node, f"{table0_name},{table1_name}"): + + try: + with When("I grant CREATE TABLE privilege"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") + + with And("I grant INSERT privilege"): + node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") + + with Then("I attempt to create table as the user"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], + exitcode=exitcode, message=message) + + for permutation in permutations(table_count=2): + with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: + + with When(f"permutation={permutation}, tables granted = {tables_granted}"): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Then("I attempt to create a table as the user"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], + exitcode=exitcode, message=message) + + with When("I grant select on all tables"): + with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Then("I attempt to create a table as the user"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestScenario +def create_with_join_union_subquery_privilege_granted_directly_or_via_role(self, node=None): + """Check that user is able to create a table with a stored query that includes `UNION ALL`, `JOIN` and two subqueries + if and only if the user has SELECT privilege on all of the tables, either granted directly or through a role. + """ + user_name = f"user_{getuid()}" + role_name = f"role_{getuid()}" + + if node is None: + node = self.context.node + + with user(node, f"{user_name}"): + + Scenario(test=create_with_join_union_subquery, + name="create with join union subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + + with user(node, f"{user_name}"), role(node, f"{role_name}"): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + Scenario(test=create_with_join_union_subquery, + name="create with join union subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + +@TestOutline +def create_with_join_union_subquery(self, grant_target_name, user_name, node=None): + """Grant select and create table privileges and check that user is able to create a table + if and only if they have all necessary privileges. + """ + table_name = f"table_{getuid()}" + table0_name = f"table0_{getuid()}" + table1_name = f"table1_{getuid()}" + table2_name = f"table2_{getuid()}" + table3_name = f"table3_{getuid()}" + table4_name = f"table4_{getuid()}" + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + create_table_query = "CREATE TABLE {table_name} ENGINE = Memory AS SELECT y FROM {table0_name} JOIN {table1_name} USING y UNION ALL SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table3_name} WHERE y IN (SELECT y FROM {table4_name} WHERE y<2))" + + if node is None: + node = self.context.node + + with table(node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}"): + with user(node, f"{user_name}"): + + try: + with When("I grant CREATE TABLE privilege"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") + + with And("I grant INSERT privilege"): + node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") + + with Then("I attempt to create table as the user with CREATE TABLE privilege"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), + settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + + for permutation in permutations(table_count=5): + with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table3_name, table4_name) as tables_granted: + + with When(f"permutation={permutation}, tables granted = {tables_granted}"): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Then("I attempt to create a table as the user"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), + settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + + with When("I grant select on all tables"): + with grant_select_on_table(node, max(permutations(table_count=5))+1, grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Then("I attempt to create a table as the user"): + node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), + settings = [("user", f"{user_name}")]) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestScenario +def create_with_nested_tables_privilege_granted_directly_or_via_role(self, node=None): + """Check that user is able to create a table with a stored query that includes other tables if and only if + they have SELECT privilege on all the tables and the source tables for those tables. + """ + user_name = f"user_{getuid()}" + role_name = f"role_{getuid()}" + + if node is None: + node = self.context.node + + with user(node, f"{user_name}"): + + Scenario(test=create_with_nested_tables, + name="create with nested tables, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + + with user(node, f"{user_name}"), role(node, f"{role_name}"): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + Scenario(test=create_with_nested_tables, + name="create with nested tables, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + +@TestOutline +def create_with_nested_tables(self, grant_target_name, user_name, node=None): + """Grant select and create table privileges and check that user is able to create a table + if and only if they have all necessary privileges. + """ + table_name = f"table_{getuid()}" + table0_name = f"table0_{getuid()}" + table1_name = f"table1_{getuid()}" + table2_name = f"table2_{getuid()}" + table3_name = f"table3_{getuid()}" + table4_name = f"table4_{getuid()}" + table5_name = f"table5_{getuid()}" + table6_name = f"table6_{getuid()}" + + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + create_table_query = "CREATE TABLE {table_name} ENGINE = Memory AS SELECT y FROM {table6_name} UNION ALL SELECT y FROM {table5_name}" + + if node is None: + node = self.context.node + + with table(node, f"{table0_name},{table2_name},{table4_name},{table6_name}"): + + try: + with Given("I have some tables"): + node.query(f"CREATE TABLE {table1_name} ENGINE = Memory AS SELECT y FROM {table0_name}") + node.query(f"CREATE TABLE {table3_name} ENGINE = Memory AS SELECT y FROM {table2_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)") + node.query(f"CREATE TABLE {table5_name} ENGINE = Memory AS SELECT y FROM {table4_name} JOIN {table3_name} USING y") + + with When("I grant CREATE TABLE privilege"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") + + with And("I grant INSERT privilege"): + node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") + + with Then("I attempt to create table as the user with CREATE TABLE privilege"): + node.query(create_table_query.format(table_name=table_name, table5_name=table5_name, table6_name=table6_name), + settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + + for permutation in ([0,1,2,3,7,11,15,31,39,79,95],permutations(table_count=7))[self.context.stress]: + with grant_select_on_table(node, permutation, grant_target_name, table5_name, table6_name, table3_name, table4_name, table1_name, table2_name, table0_name) as tables_granted: + + with When(f"permutation={permutation}, tables granted = {tables_granted}"): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table3_name}") + + with Then("I attempt to create a table as the user"): + node.query(create_table_query.format(table_name=table_name, table5_name=table5_name, table6_name=table6_name), + settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + + with When("I grant select on all tables"): + with grant_select_on_table(node, max(permutations(table_count=7))+1, + grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name, table5_name, table6_name): + + with Given("I don't have a table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Then("I attempt to create a table as the user"): + node.query(create_table_query.format(table_name=table_name, table5_name=table5_name, table6_name=table6_name), + settings = [("user", f"{user_name}")]) + + finally: + with Finally("I drop the tables"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestScenario +def create_as_another_table(self, node=None): + """Check that user is able to create a table as another table with only CREATE TABLE privilege. + """ + user_name = f"user_{getuid()}" + table_name = f"table_{getuid()}" + source_table_name = f"source_table_{getuid()}" + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + if node is None: + node = self.context.node + + with table(node, f"{source_table_name}"): + with user(node, f"{user_name}"): + + try: + with When("I grant CREATE TABLE privilege to a user"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {user_name}") + + with Then("I try to create a table as another table"): + node.query(f"CREATE TABLE {table_name} AS {source_table_name}", settings = [("user", f"{user_name}")]) + + finally: + with Finally("I drop the tables"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestScenario +def create_as_numbers(self, node=None): + """Check that user is able to create a table as numbers table function. + """ + user_name = f"user_{getuid()}" + table_name = f"table_{getuid()}" + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + if node is None: + node = self.context.node + + with user(node, f"{user_name}"): + + try: + with When("I grant CREATE TABLE privilege to a user"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {user_name}") + + with And("I grant INSERT privilege"): + node.query(f"GRANT INSERT ON {table_name} TO {user_name}") + + with Then("I try to create a table without select privilege on the table"): + node.query(f"CREATE TABLE {table_name} AS numbers(5)", settings = [("user", f"{user_name}")]) + + finally: + with Finally("I drop the tables"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestScenario +def create_as_merge(self, node=None): + """Check that user is able to create a table as merge table function. + """ + user_name = f"user_{getuid()}" + table_name = f"table_{getuid()}" + source_table_name = f"source_table_{getuid()}" + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + if node is None: + node = self.context.node + + with table(node, f"{source_table_name}"): + with user(node, f"{user_name}"): + + try: + with When("I grant CREATE TABLE privilege to a user"): + node.query(f"GRANT CREATE TABLE ON {table_name} TO {user_name}") + + with Then("I try to create a table as another table"): + node.query(f"CREATE TABLE {table_name} AS merge(default,'{source_table_name}')", settings = [("user", f"{user_name}")]) + + finally: + with Finally("I drop the tables"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateTable("1.0"), + RQ_SRS_006_RBAC_Privileges_CreateTable_Access("1.0"), +) +@Name("create table") +def feature(self, stress=None, parallel=None, node="clickhouse1"): + self.context.node = self.context.cluster.node(node) + + if stress is not None: + self.context.stress = stress + if parallel is not None: + self.context.stress = parallel + + tasks = [] + pool = Pool(10) + + try: + try: + for scenario in loads(current_module(), Scenario): + run_scenario(pool, tasks, scenario) + finally: + join(tasks) + finally: + pool.close() diff --git a/tests/testflows/rbac/tests/privileges/create/create_temp_table.py b/tests/testflows/rbac/tests/privileges/create/create_temp_table.py new file mode 100644 index 00000000000..52eeb41dd45 --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/create/create_temp_table.py @@ -0,0 +1,98 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateTemporaryTable_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute CREATE TEMPORARY TABLE when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute CREATE TEMPORARY TABLE with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE TEMPORARY TABLE with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + temp_table_name = f"temp_table_{getuid()}" + + try: + with When("I attempt to create a temporary table without privilege"): + node.query(f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the temporary table"): + node.query(f"DROP TEMPORARY TABLE IF EXISTS {temp_table_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + temp_table_name = f"temp_table_{getuid()}" + + try: + with When("I grant create temporary table privilege"): + node.query(f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}") + + with Then("I attempt to create aa temporary table"): + node.query(f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)]) + + finally: + with Finally("I drop the temporary table"): + node.query(f"DROP TEMPORARY TABLE IF EXISTS {temp_table_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + temp_table_name = f"temp_table_{getuid()}" + + try: + with When("I grant the create temporary table privilege"): + node.query(f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}") + + with And("I revoke the create temporary table privilege"): + node.query(f"REVOKE CREATE TEMPORARY TABLE ON *.* FROM {grant_target_name}") + + with Then("I attempt to create a temporary table"): + node.query(f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the temporary table"): + node.query(f"DROP TEMPORARY TABLE IF EXISTS {temp_table_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_CreateTemporaryTable("1.0"), +) +@Name("create temporary table") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of CREATE TEMPORARY TABLE. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/detach/__init__.py b/tests/testflows/rbac/tests/privileges/detach/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/rbac/tests/privileges/detach/detach_database.py b/tests/testflows/rbac/tests/privileges/detach/detach_database.py new file mode 100644 index 00000000000..d6b56f2bb25 --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/detach/detach_database.py @@ -0,0 +1,106 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_DropDatabase_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute DETACH DATABASE when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute DETACH DATABASE with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute DETACH DATABASE with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with Given("I have a database"): + node.query(f"CREATE DATABASE {db_name}") + + with When("I attempt to detach the database"): + node.query(f"DETACH DATABASE {db_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with Given("I have a database"): + node.query(f"CREATE DATABASE {db_name}") + + with When("I grant drop database privilege"): + node.query(f"GRANT DROP DATABASE ON {db_name}.* TO {grant_target_name}") + + with Then("I attempt to detach a database"): + node.query(f"DETACH DATABASE {db_name}", settings = [("user", user_name)]) + + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with Given("I have a database"): + node.query(f"CREATE DATABASE {db_name}") + + with When("I grant the drop database privilege"): + node.query(f"GRANT DROP DATABASE ON {db_name}.* TO {grant_target_name}") + + with And("I revoke the drop database privilege"): + node.query(f"REVOKE DROP DATABASE ON {db_name}.* FROM {grant_target_name}") + + with Then("I attempt to detach a database"): + node.query(f"DETACH DATABASE {db_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_DetachDatabase("1.0"), +) +@Name("detach database") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of DETACH DATABASE. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/detach/detach_dictionary.py b/tests/testflows/rbac/tests/privileges/detach/detach_dictionary.py new file mode 100644 index 00000000000..a8aab58a567 --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/detach/detach_dictionary.py @@ -0,0 +1,105 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_DropDictionary_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute DETACH DICTIONARY when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute DETACH DICTIONARY with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute DETACH DICTIONARY with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with Given("I have a dictionary"): + node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + + with When("I attempt to detach a dictionary without privilege"): + node.query(f"DETACH DICTIONARY {dict_name}", settings = [("user", user_name)], exitcode=exitcode, message=message) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with Given("I have a dictionary"): + node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + + with When("I grant drop dictionary privilege"): + node.query(f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}") + + with Then("I attempt to detach a dictionary"): + node.query(f"DETACH DICTIONARY {dict_name}", settings = [("user", user_name)]) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with Given("I have a dictionary"): + node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + + with When("I grant the drop dictionary privilege"): + node.query(f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}") + + with And("I revoke the drop dictionary privilege"): + node.query(f"REVOKE DROP DICTIONARY ON {dict_name} FROM {grant_target_name}") + + with Then("I attempt to detach a dictionary"): + node.query(f"DETACH DICTIONARY {dict_name}", settings = [("user", user_name)], exitcode=exitcode, message=message) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_DetachDictionary("1.0"), +) +@Name("detach dictionary") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of DETACH DICTIONARY. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/detach/detach_table.py b/tests/testflows/rbac/tests/privileges/detach/detach_table.py new file mode 100644 index 00000000000..07f21336a40 --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/detach/detach_table.py @@ -0,0 +1,107 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_DropTable_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute DETACH TABLE when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute DETACH TABLE with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute DETACH TABLE with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + table_name = f"table_{getuid()}" + + try: + with Given("I have a table"): + node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE=Memory") + + with When("I attempt to detach a table without privilege"): + node.query(f"DETACH TABLE {table_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + table_name = f"table_{getuid()}" + + try: + with Given("I have a table"): + node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE=Memory") + + with When("I grant drop table privilege"): + node.query(f"GRANT DROP TABLE ON *.* TO {grant_target_name}") + + with Then("I attempt to detach a table"): + node.query(f"DETACH TABLE {table_name}", settings = [("user", user_name)]) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + table_name = f"table_{getuid()}" + + try: + with Given("I have a table"): + node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE=Memory") + + with When("I grant the drop table privilege"): + node.query(f"GRANT DROP TABLE ON *.* TO {grant_target_name}") + + with And("I revoke the drop table privilege"): + node.query(f"REVOKE DROP TABLE ON *.* FROM {grant_target_name}") + + with Then("I attempt to detach a table"): + node.query(f"DETACH TABLE {table_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_DetachTable("1.0"), +) +@Name("detach table") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of DETACH TABLE. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/detach/detach_view.py b/tests/testflows/rbac/tests/privileges/detach/detach_view.py new file mode 100644 index 00000000000..667a5f8975d --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/detach/detach_view.py @@ -0,0 +1,107 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_DropView_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute DETACH VIEW when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute DETACH VIEW with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute DETACH VIEW with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + view_name = f"view_{getuid()}" + + try: + with Given("I have a view"): + node.query(f"CREATE VIEW {view_name} AS SELECT 1") + + with When("I attempt to drop a view without privilege"): + node.query(f"DETACH VIEW {view_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the view"): + node.query(f"DROP VIEW IF EXISTS {view_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + view_name = f"view_{getuid()}" + + try: + with Given("I have a view"): + node.query(f"CREATE VIEW {view_name} AS SELECT 1") + + with When("I grant drop view privilege"): + node.query(f"GRANT DROP VIEW ON {view_name} TO {grant_target_name}") + + with Then("I attempt to drop a view"): + node.query(f"DETACH VIEW {view_name}", settings = [("user", user_name)]) + + finally: + with Finally("I drop the view"): + node.query(f"DROP VIEW IF EXISTS {view_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + view_name = f"view_{getuid()}" + + try: + with Given("I have a view"): + node.query(f"CREATE VIEW {view_name} AS SELECT 1") + + with When("I grant the drop view privilege"): + node.query(f"GRANT DROP VIEW ON {view_name} TO {grant_target_name}") + + with And("I revoke the drop view privilege"): + node.query(f"REVOKE DROP VIEW ON {view_name} FROM {grant_target_name}") + + with Then("I attempt to drop a view"): + node.query(f"DETACH VIEW {view_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the view"): + node.query(f"DROP VIEW IF EXISTS {view_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_DetachView("1.0"), +) +@Name("detach view") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of DETACH VIEW. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/distributed_table.py b/tests/testflows/rbac/tests/privileges/distributed_table.py index 94b27465191..39c95c73f37 100755 --- a/tests/testflows/rbac/tests/privileges/distributed_table.py +++ b/tests/testflows/rbac/tests/privileges/distributed_table.py @@ -82,7 +82,7 @@ def create_without_privilege(self, node=None): table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -127,7 +127,7 @@ def create_with_privilege(self, user_name, grant_target_name, node=None): table1_name = f"table1_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -183,7 +183,7 @@ def select_without_privilege(self, node=None): table1_name = f"table1_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -239,7 +239,7 @@ def select_with_privilege(self, user_name, grant_target_name, node=None): table1_name = f"table1_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -297,7 +297,7 @@ def insert_without_privilege(self, node=None): table1_name = f"table1_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -353,7 +353,7 @@ def insert_with_privilege(self, user_name, grant_target_name, node=None): table1_name = f"table1_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -444,7 +444,7 @@ def select_with_table_on_materialized_view(self, user_name, grant_target_name, n view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -519,7 +519,7 @@ def select_with_table_on_source_table_of_materialized_view(self, user_name, gran view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -594,7 +594,7 @@ def select_with_table_on_distributed_table(self, user_name, grant_target_name, n table2_name = f"table1_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -663,7 +663,7 @@ def insert_with_table_on_materialized_view(self, user_name, grant_target_name, n view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -740,7 +740,7 @@ def insert_with_table_on_source_table_of_materialized_view(self, user_name, gran view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -815,7 +815,7 @@ def insert_with_table_on_distributed_table(self, user_name, grant_target_name, n table2_name = f"table1_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - cluster = self.context.cluster + cluster = self.context.cluster_name if node is None: node = self.context.node @@ -967,7 +967,7 @@ def multiple_node_user(self, node=None): def cluster_tests(self, cluster, node=None): """Scenarios to be run on different cluster configurations. """ - self.context.cluster = cluster + self.context.cluster_name = cluster tasks = [] pool = Pool(3) diff --git a/tests/testflows/rbac/tests/privileges/drop/__init__.py b/tests/testflows/rbac/tests/privileges/drop/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_database.py b/tests/testflows/rbac/tests/privileges/drop/drop_database.py new file mode 100644 index 00000000000..364399dace5 --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/drop/drop_database.py @@ -0,0 +1,106 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_DropDatabase_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute DROP DATABASE when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute DROP DATABASE with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute DROP DATABASE with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with Given("I have a database"): + node.query(f"CREATE DATABASE {db_name}") + + with When("I attempt to drop the database"): + node.query(f"DROP DATABASE {db_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with Given("I have a database"): + node.query(f"CREATE DATABASE {db_name}") + + with When("I grant drop database privilege"): + node.query(f"GRANT DROP DATABASE ON {db_name}.* TO {grant_target_name}") + + with Then("I attempt to drop a database"): + node.query(f"DROP DATABASE {db_name}", settings = [("user", user_name)]) + + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + db_name = f"db_{getuid()}" + + try: + with Given("I have a database"): + node.query(f"CREATE DATABASE {db_name}") + + with When("I grant the drop database privilege"): + node.query(f"GRANT DROP DATABASE ON {db_name}.* TO {grant_target_name}") + + with And("I revoke the drop database privilege"): + node.query(f"REVOKE DROP DATABASE ON {db_name}.* FROM {grant_target_name}") + + with Then("I attempt to drop a database"): + node.query(f"DROP DATABASE {db_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the database"): + node.query(f"DROP DATABASE IF EXISTS {db_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_DropDatabase("1.0"), +) +@Name("drop database") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of DROP DATABASE. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_dictionary.py b/tests/testflows/rbac/tests/privileges/drop/drop_dictionary.py new file mode 100644 index 00000000000..b73b3c6629b --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/drop/drop_dictionary.py @@ -0,0 +1,102 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_DropDictionary_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute DROP DICTIONARY when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + with When(f"I run checks that {user_name} is only able to execute DROP DICTIONARY with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + with And(f"I run checks that {user_name} with {role_name} is only able to execute DROP DICTIONARY with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with Given("I have a dictionary"): + node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + + with When("I attempt to drop a dictionary without privilege"): + node.query(f"DROP DICTIONARY {dict_name}", settings = [("user", user_name)], exitcode=exitcode, message=message) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with Given("I have a dictionary"): + node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + + with When("I grant drop dictionary privilege"): + node.query(f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}") + + with Then("I attempt to drop aa dictionary"): + node.query(f"DROP DICTIONARY {dict_name}", settings = [("user", user_name)]) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + dict_name = f"dict_{getuid()}" + + try: + with Given("I have a dictionary"): + node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + + with When("I grant the drop dictionary privilege"): + node.query(f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}") + + with And("I revoke the drop dictionary privilege"): + node.query(f"REVOKE DROP DICTIONARY ON {dict_name} FROM {grant_target_name}") + + with Then("I attempt to drop a dictionary"): + node.query(f"DROP DICTIONARY {dict_name}", settings = [("user", user_name)], exitcode=exitcode, message=message) + + finally: + with Finally("I drop the dictionary"): + node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_DropDictionary("1.0"), +) +@Name("drop dictionary") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of DROP DICTIONARY. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_table.py b/tests/testflows/rbac/tests/privileges/drop/drop_table.py new file mode 100644 index 00000000000..4694f16d12b --- /dev/null +++ b/tests/testflows/rbac/tests/privileges/drop/drop_table.py @@ -0,0 +1,106 @@ +from rbac.requirements import * +from rbac.helper.common import * +import rbac.helper.errors as errors + +@TestSuite +@Requirements( + RQ_SRS_006_RBAC_Privileges_DropTable_Access("1.0"), +) +def privilege_granted_directly_or_via_role(self, node=None): + """Check that user is only able to execute DROP TABLE when they have required privilege, either directly or via role. + """ + role_name = f"role_{getuid()}" + user_name = f"user_{getuid()}" + + if node is None: + node = self.context.node + + with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): + with user(node, user_name): + + with When(f"I run checks that {user_name} is only able to execute DROP TABLE with required privileges"): + privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + + with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): + with user(node, user_name), role(node, role_name): + + with When("I grant the role to the user"): + node.query(f"GRANT {role_name} TO {user_name}") + + with And(f"I run checks that {user_name} with {role_name} is only able to execute DROP TABLE with required privileges"): + privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + +def privilege_check(grant_target_name, user_name, node=None): + """Run scenarios to check the user's access with different privileges. + """ + exitcode, message = errors.not_enough_privileges(name=f"{user_name}") + + with Scenario("user without privilege", setup=instrument_clickhouse_server_log): + table_name = f"table_{getuid()}" + + try: + with Given("I have a table"): + node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE=Memory") + + with When("I attempt to drop a table without privilege"): + node.query(f"DROP TABLE {table_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Scenario("user with privilege", setup=instrument_clickhouse_server_log): + table_name = f"table_{getuid()}" + + try: + with Given("I have a table"): + node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE=Memory") + + with When("I grant drop table privilege"): + node.query(f"GRANT DROP TABLE ON *.* TO {grant_target_name}") + + with Then("I attempt to drop a table"): + node.query(f"DROP TABLE {table_name}", settings = [("user", user_name)]) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + + with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + table_name = f"table_{getuid()}" + try: + with Given("I have a table"): + node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE=Memory") + + with When("I grant the drop table privilege"): + node.query(f"GRANT DROP TABLE ON *.* TO {grant_target_name}") + + with And("I revoke the drop table privilege"): + node.query(f"REVOKE DROP TABLE ON *.* FROM {grant_target_name}") + + with Then("I attempt to drop a table"): + node.query(f"DROP TABLE {table_name}", settings = [("user", user_name)], + exitcode=exitcode, message=message) + + finally: + with Finally("I drop the table"): + node.query(f"DROP TABLE IF EXISTS {table_name}") + +@TestFeature +@Requirements( + RQ_SRS_006_RBAC_Privileges_DropTable("1.0"), +) +@Name("drop table") +def feature(self, node="clickhouse1", stress=None, parallel=None): + """Check the RBAC functionality of DROP TABLE. + """ + self.context.node = self.context.cluster.node(node) + + if parallel is not None: + self.context.parallel = parallel + if stress is not None: + self.context.stress = stress + + with Suite(test=privilege_granted_directly_or_via_role): + privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/feature.py b/tests/testflows/rbac/tests/privileges/feature.py index bc63824d322..baf465825f1 100755 --- a/tests/testflows/rbac/tests/privileges/feature.py +++ b/tests/testflows/rbac/tests/privileges/feature.py @@ -16,6 +16,8 @@ def feature(self): run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.show_tables", "feature"), flags=TE), {}) run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.public_tables", "feature"), flags=TE), {}) run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.distributed_table", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.grant_option", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.alter.alter_column", "feature"), flags=TE), {}) run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.alter.alter_index", "feature"), flags=TE), {}) run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.alter.alter_constraint", "feature"), flags=TE), {}) @@ -26,7 +28,25 @@ def feature(self): run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.alter.alter_freeze", "feature"), flags=TE), {}) run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.alter.alter_fetch", "feature"), flags=TE), {}) run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.alter.alter_move", "feature"), flags=TE), {}) - run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.grant_option", "feature"), flags=TE), {}) + + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.create.create_database", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.create.create_dictionary", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.create.create_temp_table", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.create.create_table", "feature"), flags=TE), {}) + + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.attach.attach_database", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.attach.attach_dictionary", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.attach.attach_temp_table", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.attach.attach_table", "feature"), flags=TE), {}) + + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.drop.drop_database", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.drop.drop_dictionary", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.drop.drop_table", "feature"), flags=TE), {}) + + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.detach.detach_database", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.detach.detach_dictionary", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.detach.detach_table", "feature"), flags=TE), {}) + run_scenario(pool, tasks, Feature(test=load("rbac.tests.privileges.detach.detach_view", "feature"), flags=TE), {}) finally: join(tasks) finally: From 9961182e86732f155a2193ce3c7c6286f37daccd Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 9 Nov 2020 20:31:51 +0800 Subject: [PATCH 039/258] Fix verbatim partition pruner --- src/Storages/MergeTree/KeyCondition.cpp | 7 +++++++ .../0_stateless/01540_verbatim_partition_pruning.reference | 1 + .../0_stateless/01540_verbatim_partition_pruning.sql | 7 +++++++ 3 files changed, 15 insertions(+) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 4caeafc093f..b89c68617b7 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -714,6 +714,13 @@ bool KeyCondition::canConstantBeWrappedByFunctions( break; } } + + // Check if we have enough columns to fulfill the action. + for (const auto & name : action.getNeededColumns()) + { + if (!transform.has(name)) + return false; + } action.execute(transform, true); } diff --git a/tests/queries/0_stateless/01540_verbatim_partition_pruning.reference b/tests/queries/0_stateless/01540_verbatim_partition_pruning.reference index 90888a9eaf5..b9338e6a9c4 100644 --- a/tests/queries/0_stateless/01540_verbatim_partition_pruning.reference +++ b/tests/queries/0_stateless/01540_verbatim_partition_pruning.reference @@ -1,3 +1,4 @@ 2 3 9 5 8 4 +1 2 3 diff --git a/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql b/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql index 2ef9c9e8917..16ab51d1160 100644 --- a/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql +++ b/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql @@ -21,3 +21,10 @@ select * from xy where intHash64(x) % 2 = intHash64(2) % 2; select * from xy where x = 8; drop table if exists xy; + +-- Test if we provide enough columns to generate a partition value +drop table if exists xyz; +create table xyz(x int, y int, z int) engine MergeTree partition by if(toUInt8(x), y, z) order by x settings index_granularity = 1; +insert into xyz values (1, 2, 3); +select * from xyz where y = 2; +drop table if exists xyz; From bef6463cb4546ac3e1ec0c1d8171fc791675d6e0 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Fri, 13 Nov 2020 18:50:50 +0300 Subject: [PATCH 040/258] Implement tcpPort() function --- .../registerFunctionsMiscellaneous.cpp | 2 + src/Functions/tcpPort.cpp | 49 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 src/Functions/tcpPort.cpp diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 86c06b47b1d..2c0baa7bba4 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -65,6 +65,7 @@ void registerFunctionGlobalVariable(FunctionFactory &); void registerFunctionHasThreadFuzzer(FunctionFactory &); void registerFunctionInitializeAggregation(FunctionFactory &); void registerFunctionErrorCodeToName(FunctionFactory &); +void registerFunctionTcpPort(FunctionFactory &); #if USE_ICU void registerFunctionConvertCharset(FunctionFactory &); @@ -130,6 +131,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionHasThreadFuzzer(factory); registerFunctionInitializeAggregation(factory); registerFunctionErrorCodeToName(factory); + registerFunctionTcpPort(factory); #if USE_ICU registerFunctionConvertCharset(factory); diff --git a/src/Functions/tcpPort.cpp b/src/Functions/tcpPort.cpp new file mode 100644 index 00000000000..bd7ab0361e2 --- /dev/null +++ b/src/Functions/tcpPort.cpp @@ -0,0 +1,49 @@ +#include +#include + + +namespace DB +{ + +namespace +{ + +class FunctionTcpPort : public IFunction +{ +public: + static constexpr auto name = "tcpPort"; + + static FunctionPtr create(const Context & context) + { + return std::make_shared(context.getTCPPort()); + } + + explicit FunctionTcpPort(UInt16 port_) : port(port_) + { + } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 0; } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override { return std::make_shared(); } + + bool isDeterministic() const override { return false; } + + ColumnPtr executeImpl(ColumnsWithTypeAndName &, const DataTypePtr &, size_t input_rows_count) const override + { + return DataTypeUInt16().createColumnConst(input_rows_count, port); + } + +private: + const UInt64 port; +}; + +} + +void registerFunctionTcpPort(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} From 026f7e0a27f0a9c6a89314e5aff9e2a16f5dbf99 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 13 Nov 2020 19:31:51 +0300 Subject: [PATCH 041/258] Add 's3_max_redirects' setting --- src/Core/Settings.h | 1 + src/Disks/S3/registerDiskS3.cpp | 3 ++- src/IO/S3/PocoHTTPClient.cpp | 10 +++++++--- src/IO/S3/PocoHTTPClient.h | 10 +++++++++- src/IO/S3Common.cpp | 15 +++++++++------ src/IO/S3Common.h | 10 +++++++--- src/Storages/StorageS3.cpp | 3 ++- 7 files changed, 37 insertions(+), 15 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 580756361b1..ede3e6dbe2e 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -65,6 +65,7 @@ class IColumn; M(UInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.", 0) \ M(UInt64, connections_with_failover_max_tries, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, "The maximum number of attempts to connect to replicas.", 0) \ M(UInt64, s3_min_upload_part_size, 512*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \ + M(UInt64, s3_max_redirects, 10, "Max number of S3 redirects hops allowed.", 0) \ M(Bool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \ M(Bool, use_uncompressed_cache, true, "Whether to use the cache of uncompressed blocks.", 0) \ M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \ diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index 862fd388476..1bdd86f9f57 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -132,7 +132,8 @@ void registerDiskS3(DiskFactory & factory) uri.is_virtual_hosted_style, config.getString(config_prefix + ".access_key_id", ""), config.getString(config_prefix + ".secret_access_key", ""), - context.getRemoteHostFilter()); + context.getRemoteHostFilter(), + context.getGlobalContext()); String metadata_path = config.getString(config_prefix + ".metadata_path", context.getPath() + "disks/" + name + "/"); diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 49ccb6dc1b3..f55d95ae160 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -48,9 +49,11 @@ namespace DB::S3 PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( const Aws::Client::ClientConfiguration & cfg, - const RemoteHostFilter & remote_host_filter_) + const RemoteHostFilter & remote_host_filter_, + const Context & global_context_) : Aws::Client::ClientConfiguration(cfg) , remote_host_filter(remote_host_filter_) + , global_context(global_context_) { } @@ -81,6 +84,7 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & clientConfigu Poco::Timespan(clientConfiguration.httpRequestTimeoutMs * 1000) /// receive timeout. )) , remote_host_filter(clientConfiguration.remote_host_filter) + , global_context(clientConfiguration.global_context) { } @@ -155,10 +159,10 @@ void PocoHTTPClient::makeRequestInternal( ProfileEvents::increment(select_metric(S3MetricType::Count)); - static constexpr int max_redirect_attempts = 10; + unsigned int max_redirect_attempts = global_context.getSettingsRef().s3_max_redirects; try { - for (int attempt = 0; attempt < max_redirect_attempts; ++attempt) + for (unsigned int attempt = 0; attempt < max_redirect_attempts; ++attempt) { Poco::URI poco_uri(uri); diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 25055754519..385a5a22e48 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -11,14 +11,21 @@ namespace Aws::Http::Standard class StandardHttpResponse; } +namespace DB +{ +class Context; +} + namespace DB::S3 { struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration { const RemoteHostFilter & remote_host_filter; + const Context & global_context; - PocoHTTPClientConfiguration(const Aws::Client::ClientConfiguration & cfg, const RemoteHostFilter & remote_host_filter_); + PocoHTTPClientConfiguration(const Aws::Client::ClientConfiguration & cfg, const RemoteHostFilter & remote_host_filter_, + const Context & global_context_); void updateSchemeAndRegion(); }; @@ -48,6 +55,7 @@ private: std::function per_request_configuration; ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; + const Context & global_context; }; } diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 1304b6b5054..a69349a609b 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -164,14 +164,15 @@ namespace S3 bool is_virtual_hosted_style, const String & access_key_id, const String & secret_access_key, - const RemoteHostFilter & remote_host_filter) + const RemoteHostFilter & remote_host_filter, + const Context & global_context) { Aws::Client::ClientConfiguration cfg; if (!endpoint.empty()) cfg.endpointOverride = endpoint; - return create(cfg, is_virtual_hosted_style, access_key_id, secret_access_key, remote_host_filter); + return create(cfg, is_virtual_hosted_style, access_key_id, secret_access_key, remote_host_filter, global_context); } std::shared_ptr ClientFactory::create( // NOLINT @@ -179,11 +180,12 @@ namespace S3 bool is_virtual_hosted_style, const String & access_key_id, const String & secret_access_key, - const RemoteHostFilter & remote_host_filter) + const RemoteHostFilter & remote_host_filter, + const Context & global_context) { Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key); - PocoHTTPClientConfiguration client_configuration(cfg, remote_host_filter); + PocoHTTPClientConfiguration client_configuration(cfg, remote_host_filter, global_context); client_configuration.updateSchemeAndRegion(); @@ -201,9 +203,10 @@ namespace S3 const String & access_key_id, const String & secret_access_key, HeaderCollection headers, - const RemoteHostFilter & remote_host_filter) + const RemoteHostFilter & remote_host_filter, + const Context & global_context) { - PocoHTTPClientConfiguration client_configuration({}, remote_host_filter); + PocoHTTPClientConfiguration client_configuration({}, remote_host_filter, global_context); if (!endpoint.empty()) client_configuration.endpointOverride = endpoint; diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index d411c903676..ff422b5b511 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -19,6 +19,7 @@ namespace DB class RemoteHostFilter; struct HttpHeader; using HeaderCollection = std::vector; + class Context; } namespace DB::S3 @@ -36,14 +37,16 @@ public: bool is_virtual_hosted_style, const String & access_key_id, const String & secret_access_key, - const RemoteHostFilter & remote_host_filter); + const RemoteHostFilter & remote_host_filter, + const Context & global_context); std::shared_ptr create( Aws::Client::ClientConfiguration & cfg, bool is_virtual_hosted_style, const String & access_key_id, const String & secret_access_key, - const RemoteHostFilter & remote_host_filter); + const RemoteHostFilter & remote_host_filter, + const Context & global_context); std::shared_ptr create( const String & endpoint, @@ -51,7 +54,8 @@ public: const String & access_key_id, const String & secret_access_key, HeaderCollection headers, - const RemoteHostFilter & remote_host_filter); + const RemoteHostFilter & remote_host_filter, + const Context & global_context); private: ClientFactory(); diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index ce9ebbd53b3..caebdf7ccd0 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -218,7 +218,8 @@ StorageS3::StorageS3( credentials = Aws::Auth::AWSCredentials(std::move(settings.access_key_id), std::move(settings.secret_access_key)); client = S3::ClientFactory::instance().create( - uri_.endpoint, uri_.is_virtual_hosted_style, access_key_id_, secret_access_key_, std::move(settings.headers), context_.getRemoteHostFilter()); + uri_.endpoint, uri_.is_virtual_hosted_style, access_key_id_, secret_access_key_, std::move(settings.headers), + context_.getRemoteHostFilter(), context_.getGlobalContext()); } From 151f84d845c7f77001f35409cdce77980ab58a25 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 13 Nov 2020 19:44:32 +0100 Subject: [PATCH 042/258] Updating XFails --- tests/testflows/rbac/regression.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/testflows/rbac/regression.py b/tests/testflows/rbac/regression.py index c34e25ce918..70728aa579d 100755 --- a/tests/testflows/rbac/regression.py +++ b/tests/testflows/rbac/regression.py @@ -96,11 +96,11 @@ xfails = { [(Fail, issue_16403)], "privileges/alter move/:/:/:/:/user with revoked ALTER MOVE PARTITION privilege/": [(Fail, issue_16403)], - "/rbac/privileges/create table/create with join query privilege granted directly or via role/:": + "privileges/create table/create with join query privilege granted directly or via role/:": [(Fail, issue_14149)], - "/rbac/privileges/create table/create with join union subquery privilege granted directly or via role/:": + "privileges/create table/create with join union subquery privilege granted directly or via role/:": [(Fail, issue_14149)], - "/rbac/privileges/create table/create with nested tables privilege granted directly or via role/:": + "privileges/create table/create with nested tables privilege granted directly or via role/:": [(Fail, issue_14149)], } From 6ef93ac73cac86399d8f7a674c46f03df4cb79bd Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Mon, 16 Nov 2020 21:46:36 +0800 Subject: [PATCH 043/258] Try fix MaterializeMySQL SYNC with modify binlog_checksum --- src/Core/MySQL/MySQLClient.cpp | 17 ++++++++--- src/Core/MySQL/MySQLClient.h | 4 ++- src/Core/MySQL/MySQLReplication.cpp | 3 +- src/Core/MySQL/MySQLReplication.h | 4 +++ src/Core/tests/mysql_protocol.cpp | 6 ++-- src/Databases/MySQL/MaterializeMetadata.cpp | 24 ++++++++++++++++ src/Databases/MySQL/MaterializeMetadata.h | 3 ++ .../MySQL/MaterializeMySQLSyncThread.cpp | 27 ++++++++++++------ src/IO/MySQLBinlogEventReadBuffer.cpp | 28 +++++++++++-------- src/IO/MySQLBinlogEventReadBuffer.h | 6 ++-- .../gtest_mysql_binlog_event_read_buffer.cpp | 8 +++--- .../materialize_with_ddl.py | 21 ++++++++++++++ .../test_materialize_mysql_database/test.py | 9 ++++++ utils/check-mysql-binlog/main.cpp | 5 +--- 14 files changed, 126 insertions(+), 39 deletions(-) diff --git a/src/Core/MySQL/MySQLClient.cpp b/src/Core/MySQL/MySQLClient.cpp index 9cb21a2d39a..f65fbe62274 100644 --- a/src/Core/MySQL/MySQLClient.cpp +++ b/src/Core/MySQL/MySQLClient.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -132,11 +133,19 @@ void MySQLClient::ping() writeCommand(Command::COM_PING, ""); } -void MySQLClient::startBinlogDumpGTID(UInt32 slave_id, String replicate_db, String gtid_str) +void MySQLClient::setBinlogChecksum(const String & binlog_checksum) { - /// Set binlog checksum to CRC32. - String checksum = "CRC32"; - writeCommand(Command::COM_QUERY, "SET @master_binlog_checksum = '" + checksum + "'"); + replication.setChecksumSignatureLength(Poco::toUpper(binlog_checksum) == "NONE" ? 0 : 4); +} + +void MySQLClient::startBinlogDumpGTID(UInt32 slave_id, String replicate_db, String gtid_str, const String & binlog_checksum) +{ + /// Maybe CRC32 or NONE. mysqlbinlog.cc use NONE, see its below comments: + /// Make a notice to the server that this client is checksum-aware. + /// It does not need the first fake Rotate necessary checksummed. + writeCommand(Command::COM_QUERY, "SET @master_binlog_checksum = 'CRC32'"); + + setBinlogChecksum(binlog_checksum); /// Set heartbeat 1s. UInt64 period_ns = (1 * 1e9); diff --git a/src/Core/MySQL/MySQLClient.h b/src/Core/MySQL/MySQLClient.h index a31794acc42..5835e980149 100644 --- a/src/Core/MySQL/MySQLClient.h +++ b/src/Core/MySQL/MySQLClient.h @@ -29,10 +29,12 @@ public: void disconnect(); void ping(); + void setBinlogChecksum(const String & binlog_checksum); + /// Start replication stream by GTID. /// replicate_db: replication database schema, events from other databases will be ignored. /// gtid: executed gtid sets format like 'hhhhhhhh-hhhh-hhhh-hhhh-hhhhhhhhhhhh:x-y'. - void startBinlogDumpGTID(UInt32 slave_id, String replicate_db, String gtid); + void startBinlogDumpGTID(UInt32 slave_id, String replicate_db, String gtid, const String & binlog_checksum); BinlogEventPtr readOneBinlogEvent(UInt64 milliseconds = 0); Position getPosition() const { return replication.getPosition(); } diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 6ff1670777a..a33d65fcbd5 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -57,7 +57,6 @@ namespace MySQLReplication payload.readStrict(reinterpret_cast(&create_timestamp), 4); payload.readStrict(reinterpret_cast(&event_header_length), 1); assert(event_header_length == EVENT_HEADER_LENGTH); - readStringUntilEOF(event_type_header_length, payload); } @@ -745,7 +744,7 @@ namespace MySQLReplication // skip the generic response packets header flag. payload.ignore(1); - MySQLBinlogEventReadBuffer event_payload(payload); + MySQLBinlogEventReadBuffer event_payload(payload, checksum_signature_length); EventHeader event_header; event_header.parse(event_payload); diff --git a/src/Core/MySQL/MySQLReplication.h b/src/Core/MySQL/MySQLReplication.h index 394ac729d1b..bbefb368aaf 100644 --- a/src/Core/MySQL/MySQLReplication.h +++ b/src/Core/MySQL/MySQLReplication.h @@ -526,6 +526,8 @@ namespace MySQLReplication virtual BinlogEventPtr readOneEvent() = 0; virtual void setReplicateDatabase(String db) = 0; virtual void setGTIDSets(GTIDSets sets) = 0; + virtual void setChecksumSignatureLength(size_t checksum_signature_length_) = 0; + virtual ~IFlavor() override = default; }; @@ -538,12 +540,14 @@ namespace MySQLReplication BinlogEventPtr readOneEvent() override { return event; } void setReplicateDatabase(String db) override { replicate_do_db = std::move(db); } void setGTIDSets(GTIDSets sets) override { position.gtid_sets = std::move(sets); } + void setChecksumSignatureLength(size_t checksum_signature_length_) override { checksum_signature_length = checksum_signature_length_; } private: Position position; BinlogEventPtr event; String replicate_do_db; std::shared_ptr table_map; + size_t checksum_signature_length = 4; inline bool do_replicate() { return (replicate_do_db.empty() || table_map->schema == replicate_do_db); } }; diff --git a/src/Core/tests/mysql_protocol.cpp b/src/Core/tests/mysql_protocol.cpp index 9dc46891241..98555ddcfe0 100644 --- a/src/Core/tests/mysql_protocol.cpp +++ b/src/Core/tests/mysql_protocol.cpp @@ -304,7 +304,8 @@ int main(int argc, char ** argv) "user", boost::program_options::value()->default_value("root"), "master user")( "password", boost::program_options::value()->required(), "master password")( "gtid", boost::program_options::value()->default_value(""), "executed GTID sets")( - "db", boost::program_options::value()->required(), "replicate do db"); + "db", boost::program_options::value()->required(), "replicate do db")( + "binlog_checksum", boost::program_options::value()->default_value("CRC32"), "master binlog_checksum"); boost::program_options::variables_map options; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); @@ -319,6 +320,7 @@ int main(int argc, char ** argv) auto master_password = options.at("password").as(); auto gtid_sets = options.at("gtid").as(); auto replicate_db = options.at("db").as(); + auto binlog_checksum = options.at("binlog_checksum").as(); std::cerr << "Master Host: " << host << ", Port: " << port << ", User: " << master_user << ", Password: " << master_password << ", Replicate DB: " << replicate_db << ", GTID: " << gtid_sets << std::endl; @@ -328,7 +330,7 @@ int main(int argc, char ** argv) /// Connect to the master. slave.connect(); - slave.startBinlogDumpGTID(slave_id, replicate_db, gtid_sets); + slave.startBinlogDumpGTID(slave_id, replicate_db, gtid_sets, binlog_checksum); WriteBufferFromOStream cerr(std::cerr); diff --git a/src/Databases/MySQL/MaterializeMetadata.cpp b/src/Databases/MySQL/MaterializeMetadata.cpp index 3c5bfdec594..cacf03675b4 100644 --- a/src/Databases/MySQL/MaterializeMetadata.cpp +++ b/src/Databases/MySQL/MaterializeMetadata.cpp @@ -88,6 +88,29 @@ void MaterializeMetadata::fetchMasterStatus(mysqlxx::PoolWithFailover::Entry & c executed_gtid_set = (*master_status.getByPosition(4).column)[0].safeGet(); } +void MaterializeMetadata::fetchMasterVariablesValue(const mysqlxx::PoolWithFailover::Entry & connection) +{ + Block variables_header{ + {std::make_shared(), "Variable_name"}, + {std::make_shared(), "Value"} + }; + + const String & fetch_query = "SHOW VARIABLES WHERE Variable_name = 'binlog_checksum'"; + MySQLBlockInputStream variables_input(connection, fetch_query, variables_header, DEFAULT_BLOCK_SIZE); + + while (Block variables_block = variables_input.read()) + { + ColumnPtr variables_name = variables_block.getByName("Variable_name").column; + ColumnPtr variables_value = variables_block.getByName("Value").column; + + for (size_t index = 0; index < variables_block.rows(); ++index) + { + if (variables_name->getDataAt(index) == "binlog_checksum") + binlog_checksum = variables_value->getDataAt(index).toString(); + } + } +} + static Block getShowMasterLogHeader(const String & mysql_version) { if (startsWith(mysql_version, "5.")) @@ -193,6 +216,7 @@ MaterializeMetadata::MaterializeMetadata( locked_tables = true; fetchMasterStatus(connection); + fetchMasterVariablesValue(connection); connection->query("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;").execute(); connection->query("START TRANSACTION /*!40100 WITH CONSISTENT SNAPSHOT */;").execute(); diff --git a/src/Databases/MySQL/MaterializeMetadata.h b/src/Databases/MySQL/MaterializeMetadata.h index 5e77620e365..94dfc73e5df 100644 --- a/src/Databases/MySQL/MaterializeMetadata.h +++ b/src/Databases/MySQL/MaterializeMetadata.h @@ -34,10 +34,13 @@ struct MaterializeMetadata size_t data_version = 1; size_t meta_version = 2; + String binlog_checksum = "CRC32"; std::unordered_map need_dumping_tables; void fetchMasterStatus(mysqlxx::PoolWithFailover::Entry & connection); + void fetchMasterVariablesValue(const mysqlxx::PoolWithFailover::Entry & connection); + bool checkBinlogFileExists(mysqlxx::PoolWithFailover::Entry & connection, const String & mysql_version) const; void transaction(const MySQLReplication::Position & position, const std::function & fun); diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp index 7e42b2548b0..223f3466da4 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp @@ -340,7 +340,7 @@ std::optional MaterializeMySQLSyncThread::prepareSynchroniz connection->query("COMMIT").execute(); client.connect(); - client.startBinlogDumpGTID(randomNumber(), mysql_database_name, metadata.executed_gtid_set); + client.startBinlogDumpGTID(randomNumber(), mysql_database_name, metadata.executed_gtid_set, metadata.binlog_checksum); return metadata; } catch (...) @@ -624,16 +624,27 @@ void MaterializeMySQLSyncThread::onEvent(Buffers & buffers, const BinlogEventPtr metadata.transaction(position_before_ddl, [&]() { buffers.commit(global_context); }); metadata.transaction(client.getPosition(),[&](){ executeDDLAtomic(query_event); }); } - else if (receive_event->header.type != HEARTBEAT_EVENT) + else { - const auto & dump_event_message = [&]() + /// MYSQL_UNHANDLED_EVENT + if (receive_event->header.type == ROTATE_EVENT) { - WriteBufferFromOwnString buf; - receive_event->dump(buf); - return buf.str(); - }; + /// Some behaviors(such as changing the value of "binlog_checksum") rotate the binlog file. + /// To ensure that the synchronization continues, we need to handle these events + metadata.fetchMasterVariablesValue(pool.get()); + client.setBinlogChecksum(metadata.binlog_checksum); + } + else if (receive_event->header.type != HEARTBEAT_EVENT) + { + const auto & dump_event_message = [&]() + { + WriteBufferFromOwnString buf; + receive_event->dump(buf); + return buf.str(); + }; - LOG_DEBUG(log, "Skip MySQL event: \n {}", dump_event_message()); + LOG_DEBUG(log, "Skip MySQL event: \n {}", dump_event_message()); + } } } diff --git a/src/IO/MySQLBinlogEventReadBuffer.cpp b/src/IO/MySQLBinlogEventReadBuffer.cpp index 3a2aba045d3..c495cbdfd90 100644 --- a/src/IO/MySQLBinlogEventReadBuffer.cpp +++ b/src/IO/MySQLBinlogEventReadBuffer.cpp @@ -4,9 +4,12 @@ namespace DB { -MySQLBinlogEventReadBuffer::MySQLBinlogEventReadBuffer(ReadBuffer & in_) - : ReadBuffer(nullptr, 0, 0), in(in_) +MySQLBinlogEventReadBuffer::MySQLBinlogEventReadBuffer(ReadBuffer & in_, size_t checksum_signature_length_) + : ReadBuffer(nullptr, 0, 0), in(in_), checksum_signature_length(checksum_signature_length_) { + if (checksum_signature_length) + checksum_buf = new char[checksum_signature_length]; + nextIfAtEnd(); } @@ -20,15 +23,15 @@ bool MySQLBinlogEventReadBuffer::nextImpl() if (checksum_buff_size == checksum_buff_limit) { - if (likely(in.available() > CHECKSUM_CRC32_SIGNATURE_LENGTH)) + if (likely(in.available() > checksum_signature_length)) { - working_buffer = ReadBuffer::Buffer(in.position(), in.buffer().end() - CHECKSUM_CRC32_SIGNATURE_LENGTH); + working_buffer = ReadBuffer::Buffer(in.position(), in.buffer().end() - checksum_signature_length); in.ignore(working_buffer.size()); return true; } - in.readStrict(checksum_buf, CHECKSUM_CRC32_SIGNATURE_LENGTH); - checksum_buff_size = checksum_buff_limit = CHECKSUM_CRC32_SIGNATURE_LENGTH; + in.readStrict(checksum_buf, checksum_signature_length); + checksum_buff_size = checksum_buff_limit = checksum_signature_length; } else { @@ -36,17 +39,17 @@ bool MySQLBinlogEventReadBuffer::nextImpl() checksum_buf[index] = checksum_buf[checksum_buff_limit + index]; checksum_buff_size -= checksum_buff_limit; - size_t read_bytes = CHECKSUM_CRC32_SIGNATURE_LENGTH - checksum_buff_size; - in.readStrict(checksum_buf + checksum_buff_size, read_bytes); /// Minimum CHECKSUM_CRC32_SIGNATURE_LENGTH bytes - checksum_buff_size = checksum_buff_limit = CHECKSUM_CRC32_SIGNATURE_LENGTH; + size_t read_bytes = checksum_signature_length - checksum_buff_size; + in.readStrict(checksum_buf + checksum_buff_size, read_bytes); /// Minimum checksum_signature_length bytes + checksum_buff_size = checksum_buff_limit = checksum_signature_length; } if (in.eof()) return false; - if (in.available() < CHECKSUM_CRC32_SIGNATURE_LENGTH) + if (in.available() < checksum_signature_length) { - size_t left_move_size = CHECKSUM_CRC32_SIGNATURE_LENGTH - in.available(); + size_t left_move_size = checksum_signature_length - in.available(); checksum_buff_limit = checksum_buff_size - left_move_size; } @@ -60,6 +63,9 @@ MySQLBinlogEventReadBuffer::~MySQLBinlogEventReadBuffer() { /// ignore last 4 bytes nextIfAtEnd(); + + if (checksum_signature_length) + delete checksum_buf; } catch (...) { diff --git a/src/IO/MySQLBinlogEventReadBuffer.h b/src/IO/MySQLBinlogEventReadBuffer.h index e9452aa551e..c1c02b6406a 100644 --- a/src/IO/MySQLBinlogEventReadBuffer.h +++ b/src/IO/MySQLBinlogEventReadBuffer.h @@ -8,19 +8,19 @@ namespace DB class MySQLBinlogEventReadBuffer : public ReadBuffer { protected: - static const size_t CHECKSUM_CRC32_SIGNATURE_LENGTH = 4; ReadBuffer & in; + size_t checksum_signature_length; size_t checksum_buff_size = 0; size_t checksum_buff_limit = 0; - char checksum_buf[CHECKSUM_CRC32_SIGNATURE_LENGTH]; + char * checksum_buf = nullptr; bool nextImpl() override; public: ~MySQLBinlogEventReadBuffer() override; - MySQLBinlogEventReadBuffer(ReadBuffer & in_); + MySQLBinlogEventReadBuffer(ReadBuffer & in_, size_t checksum_signature_length_); }; diff --git a/src/IO/tests/gtest_mysql_binlog_event_read_buffer.cpp b/src/IO/tests/gtest_mysql_binlog_event_read_buffer.cpp index f4d39c73a7c..536e5a89ca9 100644 --- a/src/IO/tests/gtest_mysql_binlog_event_read_buffer.cpp +++ b/src/IO/tests/gtest_mysql_binlog_event_read_buffer.cpp @@ -13,7 +13,7 @@ TEST(MySQLBinlogEventReadBuffer, CheckBoundary) std::vector memory_data(index, 0x01); ReadBufferFromMemory nested_in(memory_data.data(), index); - EXPECT_THROW({ MySQLBinlogEventReadBuffer binlog_in(nested_in); }, Exception); + EXPECT_THROW({ MySQLBinlogEventReadBuffer binlog_in(nested_in, 4); }, Exception); } } @@ -23,7 +23,7 @@ TEST(MySQLBinlogEventReadBuffer, NiceBufferSize) std::vector memory_data(6, 0x01); ReadBufferFromMemory nested_in(memory_data.data(), 6); - MySQLBinlogEventReadBuffer binlog_in(nested_in); + MySQLBinlogEventReadBuffer binlog_in(nested_in, 4); binlog_in.readStrict(res, 2); ASSERT_EQ(res[0], 0x01); ASSERT_EQ(res[1], 0x01); @@ -46,7 +46,7 @@ TEST(MySQLBinlogEventReadBuffer, BadBufferSizes) } ConcatReadBuffer concat_buffer(nested_buffers); - MySQLBinlogEventReadBuffer binlog_in(concat_buffer); + MySQLBinlogEventReadBuffer binlog_in(concat_buffer, 4); binlog_in.readStrict(res, 4); for (const auto & res_byte : res) @@ -71,7 +71,7 @@ TEST(MySQLBinlogEventReadBuffer, NiceAndBadBufferSizes) } ConcatReadBuffer concat_buffer(nested_buffers); - MySQLBinlogEventReadBuffer binlog_in(concat_buffer); + MySQLBinlogEventReadBuffer binlog_in(concat_buffer, 4); binlog_in.readStrict(res, 12); for (const auto & res_byte : res) diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index b97a1563212..189e28d5233 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -432,6 +432,7 @@ def query_event_with_empty_transaction(clickhouse_node, mysql_node, service_name clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") + def select_without_columns(clickhouse_node, mysql_node, service_name): mysql_node.query("CREATE DATABASE db") mysql_node.query("CREATE TABLE db.t (a INT PRIMARY KEY, b INT)") @@ -461,3 +462,23 @@ def select_without_columns(clickhouse_node, mysql_node, service_name): clickhouse_node.query("DROP VIEW v") clickhouse_node.query("DROP DATABASE db") mysql_node.query("DROP DATABASE db") + + +def insert_with_modify_binlog_checksum(clickhouse_node, mysql_node, service_name): + mysql_node.query("CREATE DATABASE test_checksum") + mysql_node.query("CREATE TABLE test_checksum.t (a INT PRIMARY KEY, b varchar(200))") + clickhouse_node.query("CREATE DATABASE test_checksum ENGINE = MaterializeMySQL('{}:3306', 'test_checksum', 'root', 'clickhouse')".format(service_name)) + check_query(clickhouse_node, "SHOW TABLES FROM test_checksum FORMAT TSV", "t\n") + mysql_node.query("INSERT INTO test_checksum.t VALUES(1, '1111')") + check_query(clickhouse_node, "SELECT * FROM test_checksum ORDER BY a FORMAT TSV", "1\t1111\n") + + mysql_node.query("SET GLOBAL binlog_checksum=NONE") + mysql_node.query("INSERT INTO test_checksum.t VALUES(2, '2222')") + check_query(clickhouse_node, "SELECT * FROM test_checksum ORDER BY a FORMAT TSV", "1\t1111\n2\t2222\n") + + mysql_node.query("SET GLOBAL binlog_checksum=CRC32") + mysql_node.query("INSERT INTO test_checksum.t VALUES(3, '3333')") + check_query(clickhouse_node, "SELECT * FROM test_checksum ORDER BY a FORMAT TSV", "1\t1111\n2\t2222\n3\t3333\n") + + clickhouse_node.query("DROP DATABASE test_checksum") + mysql_node.query("DROP DATABASE test_checksum") diff --git a/tests/integration/test_materialize_mysql_database/test.py b/tests/integration/test_materialize_mysql_database/test.py index 6df831e1e7d..6d617ea00e2 100644 --- a/tests/integration/test_materialize_mysql_database/test.py +++ b/tests/integration/test_materialize_mysql_database/test.py @@ -151,5 +151,14 @@ def test_materialize_database_ddl_with_empty_transaction_8_0(started_cluster, st def test_select_without_columns_5_7(started_cluster, started_mysql_5_7): materialize_with_ddl.select_without_columns(clickhouse_node, started_mysql_5_7, "mysql1") + def test_select_without_columns_8_0(started_cluster, started_mysql_8_0): materialize_with_ddl.select_without_columns(clickhouse_node, started_mysql_8_0, "mysql8_0") + + +def test_insert_with_modify_binlog_checksum_5_7(started_cluster, started_mysql_5_7): + materialize_with_ddl.insert_with_modify_binlog_checksum(clickhouse_node, started_mysql_5_7, "mysql1") + + +def test_insert_with_modify_binlog_checksum_8_0(started_cluster, started_mysql_5_7): + materialize_with_ddl.insert_with_modify_binlog_checksum(clickhouse_node, started_mysql_5_7, "mysql1") diff --git a/utils/check-mysql-binlog/main.cpp b/utils/check-mysql-binlog/main.cpp index 0d831b84dce..ccdc4cd168c 100644 --- a/utils/check-mysql-binlog/main.cpp +++ b/utils/check-mysql-binlog/main.cpp @@ -18,10 +18,7 @@ static DB::MySQLReplication::BinlogEventPtr parseSingleEventBody( { DB::MySQLReplication::BinlogEventPtr event; DB::ReadBufferPtr limit_read_buffer = std::make_shared(payload, header.event_size - 19, false); - DB::ReadBufferPtr event_payload = limit_read_buffer; - - if (exist_checksum) - event_payload = std::make_shared(*limit_read_buffer); + DB::ReadBufferPtr event_payload = std::make_shared(*limit_read_buffer, exist_checksum ? 4 : 0); switch (header.type) { From b4fc2ecc0065cccce4de30ed6de055da0331b880 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Tue, 17 Nov 2020 09:41:35 +0800 Subject: [PATCH 044/258] Try fix integration test --- .../test_materialize_mysql_database/materialize_with_ddl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index 009d15a5db8..4b5355e08ff 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -496,15 +496,15 @@ def insert_with_modify_binlog_checksum(clickhouse_node, mysql_node, service_name clickhouse_node.query("CREATE DATABASE test_checksum ENGINE = MaterializeMySQL('{}:3306', 'test_checksum', 'root', 'clickhouse')".format(service_name)) check_query(clickhouse_node, "SHOW TABLES FROM test_checksum FORMAT TSV", "t\n") mysql_node.query("INSERT INTO test_checksum.t VALUES(1, '1111')") - check_query(clickhouse_node, "SELECT * FROM test_checksum ORDER BY a FORMAT TSV", "1\t1111\n") + check_query(clickhouse_node, "SELECT * FROM test_checksum.t ORDER BY a FORMAT TSV", "1\t1111\n") mysql_node.query("SET GLOBAL binlog_checksum=NONE") mysql_node.query("INSERT INTO test_checksum.t VALUES(2, '2222')") - check_query(clickhouse_node, "SELECT * FROM test_checksum ORDER BY a FORMAT TSV", "1\t1111\n2\t2222\n") + check_query(clickhouse_node, "SELECT * FROM test_checksum.t ORDER BY a FORMAT TSV", "1\t1111\n2\t2222\n") mysql_node.query("SET GLOBAL binlog_checksum=CRC32") mysql_node.query("INSERT INTO test_checksum.t VALUES(3, '3333')") - check_query(clickhouse_node, "SELECT * FROM test_checksum ORDER BY a FORMAT TSV", "1\t1111\n2\t2222\n3\t3333\n") + check_query(clickhouse_node, "SELECT * FROM test_checksum.t ORDER BY a FORMAT TSV", "1\t1111\n2\t2222\n3\t3333\n") clickhouse_node.query("DROP DATABASE test_checksum") mysql_node.query("DROP DATABASE test_checksum") From b4f025a5e6cfbde847d281ba8327513ac498f829 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Tue, 17 Nov 2020 17:44:03 +0800 Subject: [PATCH 045/258] trigger CI From e8492a54e0648087d1dceb50a7a3fc20de3ebd1c Mon Sep 17 00:00:00 2001 From: yangshuai Date: Tue, 17 Nov 2020 17:56:29 +0800 Subject: [PATCH 046/258] Update replication.md Correct spelling --- docs/en/engines/table-engines/mergetree-family/replication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 932facc9ddc..16c6a74d94e 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -152,7 +152,7 @@ You can specify default arguments for `Replicated` table engine in the server co ```xml /clickhouse/tables/{shard}/{database}/{table} -{replica} +{replica} ``` In this case, you can omit arguments when creating tables: From 70f898e66786fc4de56e9e30208af44885142efa Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 17 Nov 2020 15:34:31 +0300 Subject: [PATCH 047/258] Order inputs for ActionsDAG. --- src/Interpreters/ActionsDAG.cpp | 54 +++++++++++++++++++++----- src/Interpreters/ActionsDAG.h | 11 +++++- src/Interpreters/ExpressionActions.cpp | 33 ++++++++++++++-- src/Interpreters/ExpressionActions.h | 10 ++++- 4 files changed, 91 insertions(+), 17 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index a788d6f84e3..0256090abc6 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -24,7 +24,7 @@ namespace ErrorCodes ActionsDAG::ActionsDAG(const NamesAndTypesList & inputs) { for (const auto & input : inputs) - addInput(input.name, input.type); + addInput(input.name, input.type, true); } ActionsDAG::ActionsDAG(const ColumnsWithTypeAndName & inputs) @@ -32,9 +32,9 @@ ActionsDAG::ActionsDAG(const ColumnsWithTypeAndName & inputs) for (const auto & input : inputs) { if (input.column && isColumnConst(*input.column)) - addInput(input); + addInput(input, true); else - addInput(input.name, input.type); + addInput(input.name, input.type, true); } } @@ -46,6 +46,9 @@ ActionsDAG::Node & ActionsDAG::addNode(Node node, bool can_replace) auto & res = nodes.emplace_back(std::move(node)); + if (res.type == ActionType::INPUT) + inputs.emplace_back(&res); + index.replace(&res); return res; } @@ -59,17 +62,17 @@ ActionsDAG::Node & ActionsDAG::getNode(const std::string & name) return **it; } -const ActionsDAG::Node & ActionsDAG::addInput(std::string name, DataTypePtr type) +const ActionsDAG::Node & ActionsDAG::addInput(std::string name, DataTypePtr type, bool can_replace) { Node node; node.type = ActionType::INPUT; node.result_type = std::move(type); node.result_name = std::move(name); - return addNode(std::move(node)); + return addNode(std::move(node), can_replace); } -const ActionsDAG::Node & ActionsDAG::addInput(ColumnWithTypeAndName column) +const ActionsDAG::Node & ActionsDAG::addInput(ColumnWithTypeAndName column, bool can_replace) { Node node; node.type = ActionType::INPUT; @@ -77,7 +80,7 @@ const ActionsDAG::Node & ActionsDAG::addInput(ColumnWithTypeAndName column) node.result_name = std::move(column.name); node.column = std::move(column.column); - return addNode(std::move(node)); + return addNode(std::move(node), can_replace); } const ActionsDAG::Node & ActionsDAG::addColumn(ColumnWithTypeAndName column) @@ -144,6 +147,14 @@ const ActionsDAG::Node & ActionsDAG::addFunction( compilation_cache = context.getCompiledExpressionCache(); #endif + return addFunction(function, argument_names, std::move(result_name)); +} + +const ActionsDAG::Node & ActionsDAG::addFunction( + const FunctionOverloadResolverPtr & function, + const Names & argument_names, + std::string result_name) +{ size_t num_arguments = argument_names.size(); Node node; @@ -231,9 +242,8 @@ const ActionsDAG::Node & ActionsDAG::addFunction( NamesAndTypesList ActionsDAG::getRequiredColumns() const { NamesAndTypesList result; - for (const auto & node : nodes) - if (node.type == ActionType::INPUT) - result.emplace_back(node.result_name, node.result_type); + for (const auto & input : inputs) + result.emplace_back(input->result_name, input->result_type); return result; } @@ -347,6 +357,8 @@ void ActionsDAG::removeUnusedActions() } nodes.remove_if([&](const Node & node) { return visited_nodes.count(&node) == 0; }); + auto it = std::remove_if(inputs.begin(), inputs.end(), [&](const Node * node) { return visited_nodes.count(node) == 0; }); + inputs.erase(it, inputs.end()); } void ActionsDAG::addAliases(const NamesWithAliases & aliases, std::vector & result_nodes) @@ -442,6 +454,9 @@ ActionsDAGPtr ActionsDAG::clone() const for (const auto & node : index) actions->index.insert(copy_map[node]); + for (const auto & node : inputs) + actions->inputs.push_back(copy_map[node]); + return actions; } @@ -540,6 +555,7 @@ ActionsDAGPtr ActionsDAG::splitActionsBeforeArrayJoin(const NameSet & array_join std::list split_nodes; Index this_index; Index split_index; + Inputs new_inputs; struct Frame { @@ -627,6 +643,7 @@ ActionsDAGPtr ActionsDAG::splitActionsBeforeArrayJoin(const NameSet & array_join input_node.result_type = child->result_type; input_node.result_name = child->result_name; // getUniqueNameForIndex(index, child->result_name); child_data.to_this = &this_nodes.emplace_back(std::move(input_node)); + new_inputs.push_back(child_data.to_this); /// This node is needed for current action, so put it to index also. split_index.replace(child_data.to_split); @@ -658,6 +675,7 @@ ActionsDAGPtr ActionsDAG::splitActionsBeforeArrayJoin(const NameSet & array_join input_node.result_type = node.result_type; input_node.result_name = node.result_name; cur_data.to_this = &this_nodes.emplace_back(std::move(input_node)); + new_inputs.push_back(cur_data.to_this); } } } @@ -676,12 +694,28 @@ ActionsDAGPtr ActionsDAG::splitActionsBeforeArrayJoin(const NameSet & array_join if (split_actions_are_empty) return {}; + Inputs this_inputs; + Inputs split_inputs; + + for (auto * input : inputs) + { + const auto & cur = data[input]; + if (cur.to_this) + this_inputs.push_back(cur.to_this); + else + split_inputs.push_back(cur.to_split); + } + + this_inputs.insert(this_inputs.end(), new_inputs.begin(), new_inputs.end()); + index.swap(this_index); nodes.swap(this_nodes); + inputs.swap(this_inputs); auto split_actions = cloneEmpty(); split_actions->nodes.swap(split_nodes); split_actions->index.swap(split_index); + split_actions->inputs.swap(split_inputs); split_actions->settings.project_input = false; return split_actions; diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 4765456ca4f..5a5dbebdedd 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -151,6 +151,7 @@ public: }; using Nodes = std::list; + using Inputs = std::vector; struct ActionsSettings { @@ -165,6 +166,7 @@ public: private: Nodes nodes; Index index; + Inputs inputs; ActionsSettings settings; @@ -181,6 +183,7 @@ public: const Nodes & getNodes() const { return nodes; } const Index & getIndex() const { return index; } + const Inputs & getInputs() const { return inputs; } NamesAndTypesList getRequiredColumns() const; ColumnsWithTypeAndName getResultColumns() const; @@ -190,11 +193,15 @@ public: std::string dumpNames() const; std::string dumpDAG() const; - const Node & addInput(std::string name, DataTypePtr type); - const Node & addInput(ColumnWithTypeAndName column); + const Node & addInput(std::string name, DataTypePtr type, bool can_replace = false); + const Node & addInput(ColumnWithTypeAndName column, bool can_replace = false); const Node & addColumn(ColumnWithTypeAndName column); const Node & addAlias(const std::string & name, std::string alias, bool can_replace = false); const Node & addArrayJoin(const std::string & source_name, std::string result_name); + const Node & addFunction( + const FunctionOverloadResolverPtr & function, + const Names & argument_names, + std::string result_name); const Node & addFunction( const FunctionOverloadResolverPtr & function, const Names & argument_names, diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 53c08481fc2..4c332036b41 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -83,6 +83,7 @@ void ExpressionActions::linearizeActions() const auto & nodes = getNodes(); const auto & index = actions_dag->getIndex(); + const auto & inputs = actions_dag->getInputs(); std::vector data(nodes.size()); std::unordered_map reverse_index; @@ -163,11 +164,11 @@ void ExpressionActions::linearizeActions() { /// Argument for input is special. It contains the position from required columns. ExpressionActions::Argument argument; - argument.pos = required_columns.size(); + // argument.pos = required_columns.size(); argument.needed_later = !cur.parents.empty(); arguments.emplace_back(argument); - required_columns.push_back({node->result_name, node->result_type}); + //required_columns.push_back({node->result_name, node->result_type}); } actions.push_back({node, arguments, free_position}); @@ -199,6 +200,15 @@ void ExpressionActions::linearizeActions() ColumnWithTypeAndName col{node->column, node->result_type, node->result_name}; sample_block.insert(std::move(col)); } + + for (const auto * input : inputs) + { + const auto & cur = data[reverse_index[input]]; + auto pos = required_columns.size(); + actions[cur.position].arguments.front().pos = pos; + required_columns.push_back({input->result_name, input->result_type}); + input_positions[input->result_name].emplace_back(pos); + } } @@ -412,7 +422,24 @@ void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run) .num_rows = num_rows, }; - execution_context.inputs_pos.reserve(required_columns.size()); + execution_context.inputs_pos.assign(required_columns.size(), -1); + + for (size_t pos = 0; pos < block.columns(); ++pos) + { + const auto & col = block.getByPosition(pos); + auto it = input_positions.find(col.name); + if (it != input_positions.end()) + { + for (auto input_pos : it->second) + { + if (execution_context.inputs_pos[input_pos] < 0) + { + execution_context.inputs_pos[input_pos] = pos; + break; + } + } + } + } for (const auto & column : required_columns) { diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index f2f5862856b..2b1aa5e2456 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -44,10 +44,10 @@ public: struct Argument { /// Position in ExecutionContext::columns - size_t pos; + size_t pos = 0; /// True if there is another action which will use this column. /// Otherwise column will be removed. - bool needed_later; + bool needed_later = false; }; using Arguments = std::vector; @@ -63,6 +63,11 @@ public: using Actions = std::vector; + /// This map helps to find input position bu it's name. + /// Key is a view to input::result_name. + /// Result is a list because it is allowed for inputs to have same names. + using NameToInputMap = std::unordered_map>; + private: ActionsDAGPtr actions_dag; @@ -70,6 +75,7 @@ private: size_t num_columns = 0; NamesAndTypesList required_columns; + NameToInputMap input_positions; ColumnNumbers result_positions; Block sample_block; From d007e5671dceabecb6bfcb393ab17687ae2d7a07 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 17 Nov 2020 15:39:41 +0300 Subject: [PATCH 048/258] Order inputs for ActionsDAG. --- src/Interpreters/ActionsDAG.cpp | 8 ++++---- src/Interpreters/ActionsDAG.h | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 0256090abc6..d046f7d4f2c 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -21,15 +21,15 @@ namespace ErrorCodes } -ActionsDAG::ActionsDAG(const NamesAndTypesList & inputs) +ActionsDAG::ActionsDAG(const NamesAndTypesList & inputs_) { - for (const auto & input : inputs) + for (const auto & input : inputs_) addInput(input.name, input.type, true); } -ActionsDAG::ActionsDAG(const ColumnsWithTypeAndName & inputs) +ActionsDAG::ActionsDAG(const ColumnsWithTypeAndName & inputs_) { - for (const auto & input : inputs) + for (const auto & input : inputs_) { if (input.column && isColumnConst(*input.column)) addInput(input, true); diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 5a5dbebdedd..7b959ff5e29 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -178,8 +178,8 @@ public: ActionsDAG() = default; ActionsDAG(const ActionsDAG &) = delete; ActionsDAG & operator=(const ActionsDAG &) = delete; - explicit ActionsDAG(const NamesAndTypesList & inputs); - explicit ActionsDAG(const ColumnsWithTypeAndName & inputs); + explicit ActionsDAG(const NamesAndTypesList & inputs_); + explicit ActionsDAG(const ColumnsWithTypeAndName & inputs_); const Nodes & getNodes() const { return nodes; } const Index & getIndex() const { return index; } From 71d726ea214ef1a1d367ce2f9273f5f62746cdd9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 17 Nov 2020 16:20:23 +0300 Subject: [PATCH 049/258] Order inputs for ActionsDAG. --- src/Interpreters/ExpressionActions.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 4c332036b41..c1dec121c90 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -441,14 +441,6 @@ void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run) } } - for (const auto & column : required_columns) - { - ssize_t pos = -1; - if (block.has(column.name)) - pos = block.getPositionByName(column.name); - execution_context.inputs_pos.push_back(pos); - } - execution_context.columns.resize(num_columns); for (const auto & action : actions) From 1787cd89a73e37665bb586196b7514d50b771e68 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Tue, 17 Nov 2020 16:24:13 +0300 Subject: [PATCH 050/258] Implement tcpPort() function literal --- .../getDictionaryConfigurationFromAST.cpp | 33 ++++++++++++++----- .../getDictionaryConfigurationFromAST.h | 3 +- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- src/Parsers/ExpressionListParsers.cpp | 5 +-- .../01018_ddl_dictionaries_create.reference | 2 +- .../01018_ddl_dictionaries_create.sql | 2 +- 6 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index 430c1d591dd..40e86d590c4 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace DB { @@ -356,7 +357,8 @@ NamesToTypeNames buildDictionaryAttributesConfiguration( void buildConfigurationFromFunctionWithKeyValueArguments( AutoPtr doc, AutoPtr root, - const ASTExpressionList * ast_expr_list) + const ASTExpressionList * ast_expr_list, + const Context & context) { const auto & children = ast_expr_list->children; for (size_t i = 0; i != children.size(); ++i) @@ -365,19 +367,30 @@ void buildConfigurationFromFunctionWithKeyValueArguments( AutoPtr current_xml_element(doc->createElement(pair->first)); root->appendChild(current_xml_element); - if (const auto * identifier = pair->second->as(); identifier) + if (const auto * identifier = pair->second->as()) { AutoPtr value(doc->createTextNode(identifier->name())); current_xml_element->appendChild(value); } - else if (const auto * literal = pair->second->as(); literal) + else if (const auto * literal = pair->second->as()) { AutoPtr value(doc->createTextNode(getFieldAsString(literal->value))); current_xml_element->appendChild(value); } - else if (const auto * list = pair->second->as(); list) + else if (const auto * list = pair->second->as()) { - buildConfigurationFromFunctionWithKeyValueArguments(doc, current_xml_element, list); + buildConfigurationFromFunctionWithKeyValueArguments(doc, current_xml_element, list, context); + } + else if (const auto * func = pair->second->as()) + { + auto builder = FunctionFactory::instance().tryGet(func->name, context); + auto function = builder->build({}); + auto result = function->execute({}, {}, 0); + + Field value; + result->get(0, value); + AutoPtr text_value(doc->createTextNode(getFieldAsString(value))); + current_xml_element->appendChild(text_value); } else { @@ -406,13 +419,14 @@ void buildSourceConfiguration( AutoPtr doc, AutoPtr root, const ASTFunctionWithKeyValueArguments * source, - const ASTDictionarySettings * settings) + const ASTDictionarySettings * settings, + const Context & context) { AutoPtr outer_element(doc->createElement("source")); root->appendChild(outer_element); AutoPtr source_element(doc->createElement(source->name)); outer_element->appendChild(source_element); - buildConfigurationFromFunctionWithKeyValueArguments(doc, source_element, source->elements->as()); + buildConfigurationFromFunctionWithKeyValueArguments(doc, source_element, source->elements->as(), context); if (settings != nullptr) { @@ -466,7 +480,8 @@ void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs } -DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const std::string & database_) +DictionaryConfigurationPtr +getDictionaryConfigurationFromAST(const ASTCreateQuery & query, const Context & context, const std::string & database_) { checkAST(query); @@ -510,7 +525,7 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer buildPrimaryKeyConfiguration(xml_document, structure_element, complex, pk_attrs, query.dictionary_attributes_list); buildLayoutConfiguration(xml_document, current_dictionary, dictionary_layout); - buildSourceConfiguration(xml_document, current_dictionary, query.dictionary->source, query.dictionary->dict_settings); + buildSourceConfiguration(xml_document, current_dictionary, query.dictionary->source, query.dictionary->dict_settings, context); buildLifetimeConfiguration(xml_document, current_dictionary, query.dictionary->lifetime); if (query.dictionary->range) diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.h b/src/Dictionaries/getDictionaryConfigurationFromAST.h index 3038f450914..5132e3c77e0 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.h +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.h @@ -10,5 +10,6 @@ using DictionaryConfigurationPtr = Poco::AutoPtrgetObjectMetadataModificationTime(dictionary_name); database->attachDictionary(dictionary_name, DictionaryAttachInfo{query_ptr, config, modification_time}); } diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 0f06a0d2480..c1ec00befaf 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -735,6 +735,7 @@ bool ParserKeyValuePair::parseImpl(Pos & pos, ASTPtr & node, Expected & expected { ParserIdentifier id_parser; ParserLiteral literal_parser; + ParserFunction func_parser; ASTPtr identifier; ASTPtr value; @@ -742,8 +743,8 @@ bool ParserKeyValuePair::parseImpl(Pos & pos, ASTPtr & node, Expected & expected if (!id_parser.parse(pos, identifier, expected)) return false; - /// If it's not literal or identifier, than it's possible list of pairs - if (!literal_parser.parse(pos, value, expected) && !id_parser.parse(pos, value, expected)) + /// If it's neither literal, nor identifier, nor function, than it's possible list of pairs + if (!func_parser.parse(pos, value, expected) && !literal_parser.parse(pos, value, expected) && !id_parser.parse(pos, value, expected)) { ParserKeyValuePairsList kv_pairs_list; ParserToken open(TokenType::OpeningRoundBracket); diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference b/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference index 5b020911d2e..e591300eddc 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference @@ -1,5 +1,5 @@ =DICTIONARY in Ordinary DB -CREATE DICTIONARY db_01018.dict1\n(\n `key_column` UInt64 DEFAULT 0,\n `second_column` UInt8 DEFAULT 1,\n `third_column` String DEFAULT \'qqq\'\n)\nPRIMARY KEY key_column\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'database_for_dict_01018\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT()) +CREATE DICTIONARY db_01018.dict1\n(\n `key_column` UInt64 DEFAULT 0,\n `second_column` UInt8 DEFAULT 1,\n `third_column` String DEFAULT \'qqq\'\n)\nPRIMARY KEY key_column\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT tcpPort() USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'database_for_dict_01018\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT()) dict1 1 db_01018 dict1 diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql index 3261b1e61d1..1a3733fd5cb 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql @@ -32,7 +32,7 @@ CREATE DICTIONARY db_01018.dict1 third_column String DEFAULT 'qqq' ) PRIMARY KEY key_column -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict_01018')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict_01018')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()); From f897f7c93ff7be363dafe3de5a593dcb457251a1 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Tue, 17 Nov 2020 16:24:45 +0300 Subject: [PATCH 051/258] Refactor IFunction to execute with const arguments --- .../AggregateFunctionMLMethod.cpp | 32 ++++++------- .../AggregateFunctionMLMethod.h | 44 ++++++++--------- src/AggregateFunctions/IAggregateFunction.h | 4 +- src/Columns/ColumnAggregateFunction.cpp | 2 +- src/Columns/ColumnAggregateFunction.h | 2 +- src/Databases/DatabaseOrdinary.cpp | 6 +-- src/Databases/DatabaseWithDictionaries.cpp | 4 +- src/Dictionaries/DictionaryFactory.cpp | 2 +- .../tests/gtest_dictionary_configuration.cpp | 17 +++---- src/Functions/CustomWeekTransforms.h | 2 +- src/Functions/DateTimeTransforms.h | 2 +- src/Functions/FunctionBase64Conversion.h | 2 +- src/Functions/FunctionBinaryArithmetic.h | 22 ++++----- src/Functions/FunctionBitTestMany.h | 4 +- src/Functions/FunctionCustomWeekToSomething.h | 2 +- .../FunctionDateOrDateTimeAddInterval.h | 4 +- .../FunctionDateOrDateTimeToSomething.h | 2 +- src/Functions/FunctionFQDN.cpp | 2 +- src/Functions/FunctionJoinGet.cpp | 2 +- src/Functions/FunctionJoinGet.h | 2 +- src/Functions/FunctionMathBinaryFloat64.h | 2 +- src/Functions/FunctionMathConstFloat64.h | 2 +- src/Functions/FunctionMathUnary.h | 2 +- src/Functions/FunctionNumericPredicate.h | 2 +- src/Functions/FunctionStartsEndsWith.h | 4 +- src/Functions/FunctionStringOrArrayToT.h | 2 +- src/Functions/FunctionStringReplace.h | 2 +- src/Functions/FunctionStringToString.h | 2 +- src/Functions/FunctionUnaryArithmetic.h | 2 +- src/Functions/FunctionUnixTimestamp64.h | 2 +- src/Functions/FunctionsAES.h | 4 +- src/Functions/FunctionsBitmap.h | 34 ++++++------- src/Functions/FunctionsCoding.h | 34 ++++++------- src/Functions/FunctionsComparison.h | 2 +- src/Functions/FunctionsConsistentHashing.h | 4 +- src/Functions/FunctionsConversion.h | 24 +++++----- src/Functions/FunctionsEmbeddedDictionaries.h | 8 ++-- src/Functions/FunctionsExternalDictionaries.h | 48 +++++++++---------- src/Functions/FunctionsExternalModels.cpp | 4 +- src/Functions/FunctionsExternalModels.h | 2 +- src/Functions/FunctionsHashing.h | 18 +++---- src/Functions/FunctionsJSON.cpp | 2 +- src/Functions/FunctionsJSON.h | 6 +-- src/Functions/FunctionsLogical.cpp | 6 +-- src/Functions/FunctionsLogical.h | 4 +- src/Functions/FunctionsMiscellaneous.h | 4 +- .../FunctionsMultiStringFuzzySearch.h | 2 +- src/Functions/FunctionsMultiStringPosition.h | 2 +- src/Functions/FunctionsMultiStringSearch.h | 2 +- src/Functions/FunctionsRandom.h | 4 +- src/Functions/FunctionsRound.h | 6 +-- src/Functions/FunctionsStringArray.h | 12 ++--- src/Functions/FunctionsStringSearch.h | 2 +- src/Functions/FunctionsStringSearchToString.h | 2 +- src/Functions/FunctionsStringSimilarity.h | 2 +- src/Functions/IFunction.cpp | 9 ++-- src/Functions/IFunction.h | 5 +- src/Functions/IFunctionAdaptors.h | 12 ++--- src/Functions/IFunctionImpl.h | 8 ++-- src/Functions/LeastGreatestGeneric.h | 2 +- src/Functions/PerformanceAdaptors.h | 2 +- src/Functions/URL/URLHierarchy.cpp | 2 +- src/Functions/URL/URLPathHierarchy.cpp | 2 +- .../URL/extractURLParameterNames.cpp | 2 +- src/Functions/URL/extractURLParameters.cpp | 2 +- src/Functions/URL/port.cpp | 2 +- src/Functions/abtesting.cpp | 2 +- src/Functions/addressToLine.cpp | 2 +- src/Functions/addressToSymbol.cpp | 2 +- src/Functions/appendTrailingCharIfAbsent.cpp | 2 +- src/Functions/array/FunctionArrayMapped.h | 2 +- src/Functions/array/array.cpp | 2 +- src/Functions/array/arrayConcat.cpp | 2 +- src/Functions/array/arrayDistinct.cpp | 4 +- src/Functions/array/arrayElement.cpp | 48 ++++++++++--------- src/Functions/array/arrayEnumerate.cpp | 2 +- src/Functions/array/arrayEnumerateExtended.h | 4 +- src/Functions/array/arrayEnumerateRanked.h | 4 +- src/Functions/array/arrayFlatten.cpp | 2 +- src/Functions/array/arrayIndex.h | 16 +++---- src/Functions/array/arrayIntersect.cpp | 8 ++-- src/Functions/array/arrayJoin.cpp | 2 +- src/Functions/array/arrayPop.h | 2 +- src/Functions/array/arrayPush.h | 2 +- src/Functions/array/arrayReduce.cpp | 4 +- src/Functions/array/arrayReduceInRanges.cpp | 4 +- src/Functions/array/arrayResize.cpp | 2 +- src/Functions/array/arrayReverse.cpp | 4 +- src/Functions/array/arrayScalarProduct.h | 12 ++--- src/Functions/array/arraySlice.cpp | 4 +- src/Functions/array/arrayUniq.cpp | 4 +- src/Functions/array/arrayWithConstant.cpp | 2 +- src/Functions/array/arrayZip.cpp | 2 +- src/Functions/array/emptyArray.cpp | 2 +- src/Functions/array/emptyArrayToSingle.cpp | 6 +-- src/Functions/array/hasAllAny.h | 2 +- src/Functions/array/mapOp.cpp | 2 +- src/Functions/array/mapPopulateSeries.cpp | 2 +- src/Functions/array/range.cpp | 2 +- src/Functions/assumeNotNull.cpp | 2 +- src/Functions/bar.cpp | 4 +- src/Functions/bitmaskToList.cpp | 4 +- src/Functions/blockNumber.cpp | 2 +- src/Functions/blockSerializedSize.cpp | 2 +- src/Functions/blockSize.cpp | 2 +- src/Functions/buildId.cpp | 2 +- src/Functions/caseWithExpression.cpp | 2 +- src/Functions/coalesce.cpp | 2 +- src/Functions/concat.cpp | 6 +-- src/Functions/convertCharset.cpp | 2 +- src/Functions/countDigits.cpp | 2 +- src/Functions/currentDatabase.cpp | 2 +- src/Functions/currentUser.cpp | 2 +- src/Functions/dateDiff.cpp | 2 +- src/Functions/date_trunc.cpp | 2 +- src/Functions/defaultValueOfArgumentType.cpp | 2 +- src/Functions/defaultValueOfTypeName.cpp | 2 +- src/Functions/demange.cpp | 2 +- src/Functions/dumpColumnStructure.cpp | 2 +- src/Functions/errorCodeToName.cpp | 2 +- src/Functions/evalMLMethod.cpp | 2 +- src/Functions/extractAllGroups.h | 2 +- src/Functions/extractGroups.cpp | 2 +- .../extractTimeZoneFromFunctionArguments.h | 2 +- src/Functions/filesystem.cpp | 2 +- src/Functions/finalizeAggregation.cpp | 2 +- src/Functions/formatDateTime.cpp | 4 +- src/Functions/formatReadable.h | 4 +- src/Functions/formatReadableTimeDelta.cpp | 2 +- src/Functions/formatRow.cpp | 2 +- src/Functions/formatString.cpp | 2 +- src/Functions/fuzzBits.cpp | 2 +- src/Functions/generateUUIDv4.cpp | 4 +- src/Functions/geoToH3.cpp | 2 +- src/Functions/geohashDecode.cpp | 2 +- src/Functions/geohashEncode.cpp | 2 +- src/Functions/geohashesInBox.cpp | 2 +- src/Functions/getMacro.cpp | 2 +- src/Functions/getScalar.cpp | 2 +- src/Functions/getSetting.cpp | 2 +- src/Functions/getSizeOfEnumType.cpp | 2 +- src/Functions/globalVariable.cpp | 2 +- src/Functions/greatCircleDistance.cpp | 4 +- src/Functions/h3EdgeAngle.cpp | 2 +- src/Functions/h3EdgeLengthM.cpp | 2 +- src/Functions/h3GetBaseCell.cpp | 2 +- src/Functions/h3GetResolution.cpp | 2 +- src/Functions/h3HexAreaM2.cpp | 2 +- src/Functions/h3IndexesAreNeighbors.cpp | 2 +- src/Functions/h3IsValid.cpp | 2 +- src/Functions/h3ToChildren.cpp | 2 +- src/Functions/h3ToParent.cpp | 2 +- src/Functions/h3ToString.cpp | 2 +- src/Functions/h3kRing.cpp | 2 +- src/Functions/hasColumnInTable.cpp | 6 +-- src/Functions/hasThreadFuzzer.cpp | 2 +- src/Functions/hostName.cpp | 2 +- src/Functions/identity.cpp | 2 +- src/Functions/if.cpp | 30 ++++++------ src/Functions/ifNotFinite.cpp | 2 +- src/Functions/ifNull.cpp | 2 +- src/Functions/ignore.cpp | 2 +- src/Functions/in.cpp | 2 +- src/Functions/initializeAggregation.cpp | 4 +- src/Functions/isConstant.cpp | 2 +- src/Functions/isDecimalOverflow.cpp | 2 +- src/Functions/isNotNull.cpp | 2 +- src/Functions/isNull.cpp | 2 +- src/Functions/isZeroOrNull.cpp | 2 +- src/Functions/logTrace.cpp | 2 +- src/Functions/lowCardinalityIndices.cpp | 2 +- src/Functions/lowCardinalityKeys.cpp | 2 +- src/Functions/materialize.cpp | 2 +- src/Functions/multiIf.cpp | 2 +- src/Functions/neighbor.cpp | 2 +- src/Functions/normalizedQueryHash.cpp | 2 +- src/Functions/now.cpp | 2 +- src/Functions/now64.cpp | 2 +- src/Functions/nullIf.cpp | 2 +- src/Functions/pointInEllipses.cpp | 2 +- src/Functions/pointInPolygon.cpp | 8 ++-- src/Functions/randConstant.cpp | 2 +- src/Functions/randomFixedString.cpp | 4 +- src/Functions/randomPrintableASCII.cpp | 2 +- src/Functions/randomString.cpp | 4 +- src/Functions/randomStringUTF8.cpp | 2 +- src/Functions/regexpQuoteMeta.cpp | 2 +- src/Functions/reinterpretAs.cpp | 2 +- src/Functions/reinterpretAsFixedString.cpp | 2 +- src/Functions/reinterpretAsString.cpp | 2 +- src/Functions/repeat.cpp | 2 +- src/Functions/replicate.cpp | 2 +- src/Functions/reverse.cpp | 2 +- src/Functions/rowNumberInAllBlocks.cpp | 4 +- src/Functions/rowNumberInBlock.cpp | 2 +- src/Functions/runningAccumulate.cpp | 2 +- src/Functions/runningDifference.h | 4 +- src/Functions/sleep.h | 2 +- src/Functions/stringToH3.cpp | 2 +- src/Functions/substring.cpp | 2 +- src/Functions/tcpPort.cpp | 2 +- src/Functions/throwIf.cpp | 2 +- src/Functions/tid.cpp | 2 +- src/Functions/timeSlots.cpp | 2 +- src/Functions/timezone.cpp | 2 +- src/Functions/toColumnTypeName.cpp | 2 +- src/Functions/toFixedString.h | 4 +- src/Functions/toLowCardinality.cpp | 2 +- src/Functions/toNullable.cpp | 2 +- src/Functions/toStartOfInterval.cpp | 2 +- src/Functions/toTimeZone.cpp | 2 +- src/Functions/toTypeName.cpp | 2 +- src/Functions/today.cpp | 2 +- src/Functions/transform.cpp | 6 +-- src/Functions/tuple.cpp | 2 +- src/Functions/tupleElement.cpp | 2 +- src/Functions/uptime.cpp | 2 +- src/Functions/version.cpp | 2 +- src/Functions/visibleWidth.cpp | 2 +- src/Functions/yesterday.cpp | 2 +- 220 files changed, 471 insertions(+), 470 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp index 812794902df..6c5c5af2f1d 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp @@ -143,7 +143,7 @@ void LinearModelData::updateState() void LinearModelData::predict( ColumnVector::Container & container, - ColumnsWithTypeAndName & arguments, + const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, const Context & context) const @@ -264,8 +264,8 @@ void Adam::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) average_gradient[i] = average_gradient[i] * frac + adam_rhs.average_gradient[i] * rhs_frac; average_squared_gradient[i] = average_squared_gradient[i] * frac + adam_rhs.average_squared_gradient[i] * rhs_frac; } - beta1_powered_ *= adam_rhs.beta1_powered_; - beta2_powered_ *= adam_rhs.beta2_powered_; + beta1_powered *= adam_rhs.beta1_powered; + beta2_powered *= adam_rhs.beta2_powered; } void Adam::update(UInt64 batch_size, std::vector & weights, Float64 & bias, Float64 learning_rate, const std::vector & batch_gradient) @@ -282,21 +282,21 @@ void Adam::update(UInt64 batch_size, std::vector & weights, Float64 & b for (size_t i = 0; i != average_gradient.size(); ++i) { Float64 normed_gradient = batch_gradient[i] / batch_size; - average_gradient[i] = beta1_ * average_gradient[i] + (1 - beta1_) * normed_gradient; - average_squared_gradient[i] = beta2_ * average_squared_gradient[i] + - (1 - beta2_) * normed_gradient * normed_gradient; + average_gradient[i] = beta1 * average_gradient[i] + (1 - beta1) * normed_gradient; + average_squared_gradient[i] = beta2 * average_squared_gradient[i] + + (1 - beta2) * normed_gradient * normed_gradient; } for (size_t i = 0; i < weights.size(); ++i) { weights[i] += (learning_rate * average_gradient[i]) / - ((1 - beta1_powered_) * (sqrt(average_squared_gradient[i] / (1 - beta2_powered_)) + eps_)); + ((1 - beta1_powered) * (sqrt(average_squared_gradient[i] / (1 - beta2_powered)) + eps)); } bias += (learning_rate * average_gradient[weights.size()]) / - ((1 - beta1_powered_) * (sqrt(average_squared_gradient[weights.size()] / (1 - beta2_powered_)) + eps_)); + ((1 - beta1_powered) * (sqrt(average_squared_gradient[weights.size()] / (1 - beta2_powered)) + eps)); - beta1_powered_ *= beta1_; - beta2_powered_ *= beta2_; + beta1_powered *= beta1; + beta2_powered *= beta2; } void Adam::addToBatch( @@ -348,7 +348,7 @@ void Nesterov::update(UInt64 batch_size, std::vector & weights, Float64 for (size_t i = 0; i < batch_gradient.size(); ++i) { - accumulated_gradient[i] = accumulated_gradient[i] * alpha_ + (learning_rate * batch_gradient[i]) / batch_size; + accumulated_gradient[i] = accumulated_gradient[i] * alpha + (learning_rate * batch_gradient[i]) / batch_size; } for (size_t i = 0; i < weights.size(); ++i) { @@ -375,9 +375,9 @@ void Nesterov::addToBatch( std::vector shifted_weights(weights.size()); for (size_t i = 0; i != shifted_weights.size(); ++i) { - shifted_weights[i] = weights[i] + accumulated_gradient[i] * alpha_; + shifted_weights[i] = weights[i] + accumulated_gradient[i] * alpha; } - auto shifted_bias = bias + accumulated_gradient[weights.size()] * alpha_; + auto shifted_bias = bias + accumulated_gradient[weights.size()] * alpha; gradient_computer.compute(batch_gradient, shifted_weights, shifted_bias, l2_reg_coef, target, columns, row_num); } @@ -411,7 +411,7 @@ void Momentum::update(UInt64 batch_size, std::vector & weights, Float64 for (size_t i = 0; i < batch_gradient.size(); ++i) { - accumulated_gradient[i] = accumulated_gradient[i] * alpha_ + (learning_rate * batch_gradient[i]) / batch_size; + accumulated_gradient[i] = accumulated_gradient[i] * alpha + (learning_rate * batch_gradient[i]) / batch_size; } for (size_t i = 0; i < weights.size(); ++i) { @@ -448,7 +448,7 @@ void IWeightsUpdater::addToBatch( void LogisticRegression::predict( ColumnVector::Container & container, - ColumnsWithTypeAndName & arguments, + const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, const std::vector & weights, @@ -516,7 +516,7 @@ void LogisticRegression::compute( void LinearRegression::predict( ColumnVector::Container & container, - ColumnsWithTypeAndName & arguments, + const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, const std::vector & weights, diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.h b/src/AggregateFunctions/AggregateFunctionMLMethod.h index 494907c4002..b6912405fef 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.h +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.h @@ -23,7 +23,7 @@ GradientComputer class computes gradient according to its loss function class IGradientComputer { public: - IGradientComputer() {} + IGradientComputer() = default; virtual ~IGradientComputer() = default; @@ -39,7 +39,7 @@ public: virtual void predict( ColumnVector::Container & container, - ColumnsWithTypeAndName & arguments, + const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, const std::vector & weights, @@ -51,7 +51,7 @@ public: class LinearRegression : public IGradientComputer { public: - LinearRegression() {} + LinearRegression() = default; void compute( std::vector & batch_gradient, @@ -64,7 +64,7 @@ public: void predict( ColumnVector::Container & container, - ColumnsWithTypeAndName & arguments, + const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, const std::vector & weights, @@ -76,7 +76,7 @@ public: class LogisticRegression : public IGradientComputer { public: - LogisticRegression() {} + LogisticRegression() = default; void compute( std::vector & batch_gradient, @@ -89,7 +89,7 @@ public: void predict( ColumnVector::Container & container, - ColumnsWithTypeAndName & arguments, + const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, const std::vector & weights, @@ -147,9 +147,9 @@ public: class Momentum : public IWeightsUpdater { public: - Momentum() {} + Momentum() = default; - Momentum(Float64 alpha) : alpha_(alpha) {} + explicit Momentum(Float64 alpha_) : alpha(alpha_) {} void update(UInt64 batch_size, std::vector & weights, Float64 & bias, Float64 learning_rate, const std::vector & batch_gradient) override; @@ -160,7 +160,7 @@ public: void read(ReadBuffer & buf) override; private: - Float64 alpha_{0.1}; + Float64 alpha{0.1}; std::vector accumulated_gradient; }; @@ -168,9 +168,9 @@ private: class Nesterov : public IWeightsUpdater { public: - Nesterov() {} + Nesterov() = default; - Nesterov(Float64 alpha) : alpha_(alpha) {} + explicit Nesterov(Float64 alpha_) : alpha(alpha_) {} void addToBatch( std::vector & batch_gradient, @@ -191,7 +191,7 @@ public: void read(ReadBuffer & buf) override; private: - const Float64 alpha_ = 0.9; + const Float64 alpha = 0.9; std::vector accumulated_gradient; }; @@ -201,8 +201,8 @@ class Adam : public IWeightsUpdater public: Adam() { - beta1_powered_ = beta1_; - beta2_powered_ = beta2_; + beta1_powered = beta1; + beta2_powered = beta2; } void addToBatch( @@ -225,11 +225,11 @@ public: private: /// beta1 and beta2 hyperparameters have such recommended values - const Float64 beta1_ = 0.9; - const Float64 beta2_ = 0.999; - const Float64 eps_ = 0.000001; - Float64 beta1_powered_; - Float64 beta2_powered_; + const Float64 beta1 = 0.9; + const Float64 beta2 = 0.999; + const Float64 eps = 0.000001; + Float64 beta1_powered; + Float64 beta2_powered; std::vector average_gradient; std::vector average_squared_gradient; @@ -241,7 +241,7 @@ private: class LinearModelData { public: - LinearModelData() {} + LinearModelData() = default; LinearModelData( Float64 learning_rate_, @@ -261,7 +261,7 @@ public: void predict( ColumnVector::Container & container, - ColumnsWithTypeAndName & arguments, + const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, const Context & context) const; @@ -360,7 +360,7 @@ public: void predictValues( ConstAggregateDataPtr place, IColumn & to, - ColumnsWithTypeAndName & arguments, + const ColumnsWithTypeAndName & arguments, size_t offset, size_t limit, const Context & context) const override diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 4f9552d2345..87f6a11406c 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -60,7 +60,7 @@ public: throw Exception("Prediction is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED); } - virtual ~IAggregateFunction() {} + virtual ~IAggregateFunction() = default; /** Data manipulating functions. */ @@ -113,7 +113,7 @@ public: virtual void predictValues( ConstAggregateDataPtr /* place */, IColumn & /*to*/, - ColumnsWithTypeAndName & /*arguments*/, + const ColumnsWithTypeAndName & /*arguments*/, size_t /*offset*/, size_t /*limit*/, const Context & /*context*/) const diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index f5b266b6983..99b3342f314 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -161,7 +161,7 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues(MutableColumnPtr colum return res; } -MutableColumnPtr ColumnAggregateFunction::predictValues(ColumnsWithTypeAndName & arguments, const Context & context) const +MutableColumnPtr ColumnAggregateFunction::predictValues(const ColumnsWithTypeAndName & arguments, const Context & context) const { MutableColumnPtr res = func->getReturnTypeToPredict()->createColumn(); res->reserve(data.size()); diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index 79e52e5769a..4e5e66542e9 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -119,7 +119,7 @@ public: const char * getFamilyName() const override { return "AggregateFunction"; } TypeIndex getDataType() const override { return TypeIndex::AggregateFunction; } - MutableColumnPtr predictValues(ColumnsWithTypeAndName & arguments, const Context & context) const; + MutableColumnPtr predictValues(const ColumnsWithTypeAndName & arguments, const Context & context) const; size_t size() const override { diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index a1d24226444..24e2bdcd6b2 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -67,14 +67,14 @@ namespace } - void tryAttachDictionary(const ASTPtr & query, DatabaseOrdinary & database, const String & metadata_path) + void tryAttachDictionary(const ASTPtr & query, DatabaseOrdinary & database, const String & metadata_path, const Context & context) { auto & create_query = query->as(); assert(create_query.is_dictionary); try { Poco::File meta_file(metadata_path); - auto config = getDictionaryConfigurationFromAST(create_query, database.getDatabaseName()); + auto config = getDictionaryConfigurationFromAST(create_query, context, database.getDatabaseName()); time_t modification_time = meta_file.getLastModified().epochTime(); database.attachDictionary(create_query.table, DictionaryAttachInfo{query, config, modification_time}); } @@ -190,7 +190,7 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto auto create_query = query->as(); if (create_query.is_dictionary) { - tryAttachDictionary(query, *this, getMetadataPath() + name); + tryAttachDictionary(query, *this, getMetadataPath() + name, context); /// Messages, so that it's not boring to wait for the server to load for a long time. logAboutProgress(log, ++dictionaries_processed, total_dictionaries, watch); diff --git a/src/Databases/DatabaseWithDictionaries.cpp b/src/Databases/DatabaseWithDictionaries.cpp index 6c5173c986f..ee16f4ae15e 100644 --- a/src/Databases/DatabaseWithDictionaries.cpp +++ b/src/Databases/DatabaseWithDictionaries.cpp @@ -176,7 +176,7 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S /// Add a temporary repository containing the dictionary. /// We need this temp repository to try loading the dictionary before actually attaching it to the database. auto temp_repository = external_loader.addConfigRepository(std::make_unique( - getDatabaseName(), dictionary_metadata_tmp_path, getDictionaryConfigurationFromAST(query->as()))); + getDatabaseName(), dictionary_metadata_tmp_path, getDictionaryConfigurationFromAST(query->as(), context))); bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true); if (!lazy_load) @@ -186,7 +186,7 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S external_loader.load(dict_id.getInternalDictionaryName()); } - auto config = getDictionaryConfigurationFromAST(query->as()); + auto config = getDictionaryConfigurationFromAST(query->as(), context); attachDictionary(dictionary_name, DictionaryAttachInfo{query, config, time(nullptr)}); SCOPE_EXIT({ if (!succeeded) diff --git a/src/Dictionaries/DictionaryFactory.cpp b/src/Dictionaries/DictionaryFactory.cpp index c33b7b5a3ae..ad19d7c20ea 100644 --- a/src/Dictionaries/DictionaryFactory.cpp +++ b/src/Dictionaries/DictionaryFactory.cpp @@ -62,7 +62,7 @@ DictionaryPtr DictionaryFactory::create( DictionaryPtr DictionaryFactory::create(const std::string & name, const ASTCreateQuery & ast, const Context & context) const { - auto configuration = getDictionaryConfigurationFromAST(ast); + auto configuration = getDictionaryConfigurationFromAST(ast, context); return DictionaryFactory::create(name, *configuration, "dictionary", context, true); } diff --git a/src/Dictionaries/tests/gtest_dictionary_configuration.cpp b/src/Dictionaries/tests/gtest_dictionary_configuration.cpp index 62422124bd8..830d4655331 100644 --- a/src/Dictionaries/tests/gtest_dictionary_configuration.cpp +++ b/src/Dictionaries/tests/gtest_dictionary_configuration.cpp @@ -1,12 +1,13 @@ -#include -#include +#include +#include +#include #include #include #include #include #include -#include -#include +#include +#include #include @@ -47,7 +48,7 @@ TEST(ConvertDictionaryAST, SimpleDictConfiguration) ParserCreateDictionaryQuery parser; ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); - DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create); + DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create, Context::createGlobal(Context::createShared().get())); /// name EXPECT_EQ(config->getString("dictionary.database"), "test"); @@ -115,7 +116,7 @@ TEST(ConvertDictionaryAST, TrickyAttributes) ParserCreateDictionaryQuery parser; ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); - DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create); + DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create, Context::createGlobal(Context::createShared().get())); Poco::Util::AbstractConfiguration::Keys keys; config->keys("dictionary.structure", keys); @@ -160,7 +161,7 @@ TEST(ConvertDictionaryAST, ComplexKeyAndLayoutWithParams) ParserCreateDictionaryQuery parser; ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); - DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create); + DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create, Context::createGlobal(Context::createShared().get())); Poco::Util::AbstractConfiguration::Keys keys; config->keys("dictionary.structure.key", keys); @@ -211,7 +212,7 @@ TEST(ConvertDictionaryAST, ComplexSource) ParserCreateDictionaryQuery parser; ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTCreateQuery * create = ast->as(); - DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create); + DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create, Context::createGlobal(Context::createShared().get())); /// source EXPECT_EQ(config->getString("dictionary.source.mysql.host"), "localhost"); EXPECT_EQ(config->getInt("dictionary.source.mysql.port"), 9000); diff --git a/src/Functions/CustomWeekTransforms.h b/src/Functions/CustomWeekTransforms.h index 53baaff8db9..afcbadc835c 100644 --- a/src/Functions/CustomWeekTransforms.h +++ b/src/Functions/CustomWeekTransforms.h @@ -116,7 +116,7 @@ template struct CustomWeekTransformImpl { template - static ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/, Transform transform = {}) + static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/, Transform transform = {}) { const auto op = Transformer{std::move(transform)}; diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 52cc43c3847..4ad99b528ea 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -683,7 +683,7 @@ struct Transformer template struct DateTimeTransformImpl { - static ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/, const Transform & transform = {}) + static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/, const Transform & transform = {}) { using Op = Transformer; diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index adc131053e2..4bc2a779cf4 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -91,7 +91,7 @@ public: return std::make_shared(); } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const ColumnPtr column_string = arguments[0].column; const ColumnString * input = checkAndGetColumn(column_string.get()); diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 43ff42956cd..7ffdc033a00 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -613,7 +613,7 @@ class FunctionBinaryArithmetic : public IFunction } /// Multiply aggregation state by integer constant: by merging it with itself specified number of times. - ColumnPtr executeAggregateMultiply(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const + ColumnPtr executeAggregateMultiply(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const { ColumnsWithTypeAndName new_arguments = arguments; if (WhichDataType(new_arguments[1].type).isAggregateFunction()) @@ -680,7 +680,7 @@ class FunctionBinaryArithmetic : public IFunction } /// Merge two aggregation states together. - ColumnPtr executeAggregateAddition(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const + ColumnPtr executeAggregateAddition(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const { const IColumn & lhs_column = *arguments[0].column; const IColumn & rhs_column = *arguments[1].column; @@ -712,7 +712,7 @@ class FunctionBinaryArithmetic : public IFunction return column_to; } - ColumnPtr executeDateTimeIntervalPlusMinus(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, + ColumnPtr executeDateTimeIntervalPlusMinus(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, const FunctionOverloadResolverPtr & function_builder) const { ColumnsWithTypeAndName new_arguments = arguments; @@ -847,7 +847,7 @@ public: return type_res; } - ColumnPtr executeFixedString(ColumnsWithTypeAndName & arguments) const + ColumnPtr executeFixedString(const ColumnsWithTypeAndName & arguments) const { using OpImpl = FixedStringOperationImpl>; @@ -923,7 +923,7 @@ public: } template - ColumnPtr executeNumeric(ColumnsWithTypeAndName & arguments, const A & left, const B & right) const + ColumnPtr executeNumeric(const ColumnsWithTypeAndName & arguments, const A & left, const B & right) const { using LeftDataType = std::decay_t; using RightDataType = std::decay_t; @@ -1047,7 +1047,7 @@ public: return nullptr; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { /// Special case when multiply aggregate function state if (isAggregateMultiply(arguments[0].type, arguments[1].type)) @@ -1181,7 +1181,7 @@ public: { } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (left.column && isColumnConst(*left.column) && arguments.size() == 1) { @@ -1205,12 +1205,8 @@ public: bool hasInformationAboutMonotonicity() const override { - std::string_view name_ = Name::name; - if (name_ == "minus" || name_ == "plus" || name_ == "divide" || name_ == "intDiv") - { - return true; - } - return false; + std::string_view name = Name::name; + return (name == "minus" || name == "plus" || name == "divide" || name == "intDiv"); } Monotonicity getMonotonicityForRange(const IDataType &, const Field & left_point, const Field & right_point) const override diff --git a/src/Functions/FunctionBitTestMany.h b/src/Functions/FunctionBitTestMany.h index 0c8b803bd22..6d527c66390 100644 --- a/src/Functions/FunctionBitTestMany.h +++ b/src/Functions/FunctionBitTestMany.h @@ -54,7 +54,7 @@ public: return std::make_shared(); } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override { const auto * value_col = arguments.front().column.get(); @@ -75,7 +75,7 @@ public: private: template ColumnPtr execute( - ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, + const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const IColumn * const value_col_untyped) const { if (const auto value_col = checkAndGetColumn>(value_col_untyped)) diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index 74d6a2b5182..8a343cffb95 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -96,7 +96,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const IDataType * from_type = arguments[0].type.get(); WhichDataType which(from_type); diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index bf2d20ceba7..70e2616eeac 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -305,7 +305,7 @@ private: template struct DateTimeAddIntervalImpl { - static ColumnPtr execute(Transform transform, ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) + static ColumnPtr execute(Transform transform, const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) { using FromValueType = typename FromDataType::FieldType; using FromColumnType = typename FromDataType::ColumnType; @@ -463,7 +463,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {2}; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override { const IDataType * from_type = arguments[0].type.get(); WhichDataType which(from_type); diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index 2d2e4a7ad6f..e0676f3dc0f 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -95,7 +95,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const IDataType * from_type = arguments[0].type.get(); WhichDataType which(from_type); diff --git a/src/Functions/FunctionFQDN.cpp b/src/Functions/FunctionFQDN.cpp index b47675d63b4..7b3b89eb511 100644 --- a/src/Functions/FunctionFQDN.cpp +++ b/src/Functions/FunctionFQDN.cpp @@ -34,7 +34,7 @@ public: return std::make_shared(); } - ColumnPtr executeImpl(ColumnsWithTypeAndName &, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr & result_type, size_t input_rows_count) const override { return result_type->createColumnConst( input_rows_count, getFQDNOrHostName())->convertToFullColumnIfConst(); diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index 4e97951fbc0..6b15bf821b2 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -17,7 +17,7 @@ namespace ErrorCodes } template -ColumnPtr ExecutableFunctionJoinGet::execute(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) +ColumnPtr ExecutableFunctionJoinGet::execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const { ColumnsWithTypeAndName keys; for (size_t i = 2; i < arguments.size(); ++i) diff --git a/src/Functions/FunctionJoinGet.h b/src/Functions/FunctionJoinGet.h index 780b59e20f4..27f348e9698 100644 --- a/src/Functions/FunctionJoinGet.h +++ b/src/Functions/FunctionJoinGet.h @@ -24,7 +24,7 @@ public: bool useDefaultImplementationForLowCardinalityColumns() const override { return true; } bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) override; + ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override; String getName() const override { return name; } diff --git a/src/Functions/FunctionMathBinaryFloat64.h b/src/Functions/FunctionMathBinaryFloat64.h index ea222379e1c..0a0688dc75c 100644 --- a/src/Functions/FunctionMathBinaryFloat64.h +++ b/src/Functions/FunctionMathBinaryFloat64.h @@ -204,7 +204,7 @@ private: return nullptr; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnWithTypeAndName & col_left = arguments[0]; const ColumnWithTypeAndName & col_right = arguments[1]; diff --git a/src/Functions/FunctionMathConstFloat64.h b/src/Functions/FunctionMathConstFloat64.h index 42729d5e9f6..f03f469bc35 100644 --- a/src/Functions/FunctionMathConstFloat64.h +++ b/src/Functions/FunctionMathConstFloat64.h @@ -25,7 +25,7 @@ private: return std::make_shared(); } - ColumnPtr executeImpl(ColumnsWithTypeAndName &, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr & result_type, size_t input_rows_count) const override { return result_type->createColumnConst(input_rows_count, Impl::value); } diff --git a/src/Functions/FunctionMathUnary.h b/src/Functions/FunctionMathUnary.h index abf38d277f3..49b0428811a 100644 --- a/src/Functions/FunctionMathUnary.h +++ b/src/Functions/FunctionMathUnary.h @@ -148,7 +148,7 @@ private: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnWithTypeAndName & col = arguments[0]; ColumnPtr res; diff --git a/src/Functions/FunctionNumericPredicate.h b/src/Functions/FunctionNumericPredicate.h index 825a8b0de15..72a17adac4c 100644 --- a/src/Functions/FunctionNumericPredicate.h +++ b/src/Functions/FunctionNumericPredicate.h @@ -46,7 +46,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const auto * in = arguments.front().column.get(); diff --git a/src/Functions/FunctionStartsEndsWith.h b/src/Functions/FunctionStartsEndsWith.h index 612e0b3b046..2899bc259d5 100644 --- a/src/Functions/FunctionStartsEndsWith.h +++ b/src/Functions/FunctionStartsEndsWith.h @@ -63,7 +63,7 @@ public: return std::make_shared(); } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const IColumn * haystack_column = arguments[0].column.get(); const IColumn * needle_column = arguments[1].column.get(); @@ -159,7 +159,7 @@ public: #endif } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { return selector.selectAndExecute(arguments, result_type, input_rows_count); } diff --git a/src/Functions/FunctionStringOrArrayToT.h b/src/Functions/FunctionStringOrArrayToT.h index f806106560c..6330d8f90d6 100644 --- a/src/Functions/FunctionStringOrArrayToT.h +++ b/src/Functions/FunctionStringOrArrayToT.h @@ -50,7 +50,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override { const ColumnPtr column = arguments[0].column; if (const ColumnString * col = checkAndGetColumn(column.get())) diff --git a/src/Functions/FunctionStringReplace.h b/src/Functions/FunctionStringReplace.h index 4ec85591726..bd8edbf9202 100644 --- a/src/Functions/FunctionStringReplace.h +++ b/src/Functions/FunctionStringReplace.h @@ -52,7 +52,7 @@ public: return std::make_shared(); } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnPtr column_src = arguments[0].column; const ColumnPtr column_needle = arguments[1].column; diff --git a/src/Functions/FunctionStringToString.h b/src/Functions/FunctionStringToString.h index db85e85a053..4123b41c547 100644 --- a/src/Functions/FunctionStringToString.h +++ b/src/Functions/FunctionStringToString.h @@ -52,7 +52,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnPtr column = arguments[0].column; if (const ColumnString * col = checkAndGetColumn(column.get())) diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index e62781cc3a1..389c171bfce 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -154,7 +154,7 @@ public: return result; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { ColumnPtr result_column; bool valid = castType(arguments[0].type.get(), [&](const auto & type) diff --git a/src/Functions/FunctionUnixTimestamp64.h b/src/Functions/FunctionUnixTimestamp64.h index 2a5dee7734a..20e225990bd 100644 --- a/src/Functions/FunctionUnixTimestamp64.h +++ b/src/Functions/FunctionUnixTimestamp64.h @@ -65,7 +65,7 @@ public: } } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { using SourceColumnType = typename SourceDataType::ColumnType; using ResultColumnType = typename ResultDataType::ColumnType; diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 68d8b41407d..5a5c5dc05b0 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -178,7 +178,7 @@ private: return std::make_shared(); } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { using namespace OpenSSLDetails; @@ -448,7 +448,7 @@ private: return std::make_shared(); } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { using namespace OpenSSLDetails; diff --git a/src/Functions/FunctionsBitmap.h b/src/Functions/FunctionsBitmap.h index ec43ae6351f..93da4906658 100644 --- a/src/Functions/FunctionsBitmap.h +++ b/src/Functions/FunctionsBitmap.h @@ -122,7 +122,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /* input_rows_count */) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /* input_rows_count */) const override { const IDataType * from_type = arguments[0].type.get(); const auto * array_type = typeid_cast(from_type); @@ -146,7 +146,7 @@ public: private: template - ColumnPtr executeBitmapData(DataTypes & argument_types, ColumnsWithTypeAndName & arguments) const + ColumnPtr executeBitmapData(DataTypes & argument_types, const ColumnsWithTypeAndName & arguments) const { // input data const ColumnArray * array = typeid_cast(arguments[0].column.get()); @@ -207,7 +207,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { // input data const auto & return_type = result_type; @@ -240,7 +240,7 @@ private: template void executeIntType( - ColumnsWithTypeAndName & arguments, size_t input_rows_count, IColumn & res_data_col, ColumnArray::Offsets & res_offsets) + const ColumnsWithTypeAndName & arguments, size_t input_rows_count, IColumn & res_data_col, ColumnArray::Offsets & res_offsets) const { const ColumnAggregateFunction * column @@ -299,7 +299,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const IDataType * from_type = arguments[0].type.get(); const DataTypeAggregateFunction * aggr_type = typeid_cast(from_type); @@ -321,7 +321,7 @@ private: using ToType = UInt64; template - ColumnPtr executeIntType(ColumnsWithTypeAndName & arguments, size_t input_rows_count) const + ColumnPtr executeIntType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { const IColumn * column_ptrs[3]; bool is_column_const[3]; @@ -417,7 +417,7 @@ public: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); for (size_t i = 0; i < 2; ++i) { - auto array_type = typeid_cast(arguments[i + 1].get()); + const auto * array_type = typeid_cast(arguments[i + 1].get()); String msg(i == 0 ? "Second" : "Third"); msg += " argument for function " + getName() + " must be an UInt32 array but it has type " + arguments[i + 1]->getName() + "."; if (!array_type) @@ -433,7 +433,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const IDataType * from_type = arguments[0].type.get(); const DataTypeAggregateFunction * aggr_type = typeid_cast(from_type); @@ -455,7 +455,7 @@ private: using ToType = UInt64; template - ColumnPtr executeIntType(ColumnsWithTypeAndName & arguments, size_t input_rows_count) const + ColumnPtr executeIntType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { const IColumn * column_ptrs[3]; bool is_column_const[3]; @@ -565,7 +565,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { auto col_to = ColumnVector::create(input_rows_count); typename ColumnVector::Container & vec_to = col_to->getData(); @@ -593,7 +593,7 @@ private: template void executeIntType( - ColumnsWithTypeAndName & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const + const ColumnsWithTypeAndName & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const { const ColumnAggregateFunction * column = typeid_cast(arguments[0].column.get()); @@ -735,7 +735,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { auto col_to = ColumnVector::create(input_rows_count); typename ColumnVector::Container & vec_to = col_to->getData(); @@ -761,7 +761,7 @@ public: private: template void executeIntType( - ColumnsWithTypeAndName & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const + const ColumnsWithTypeAndName & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const { const IColumn * column_ptrs[2]; bool is_column_const[2]; @@ -832,7 +832,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { auto col_to = ColumnVector::create(input_rows_count); typename ColumnVector::Container & vec_to = col_to->getData(); @@ -858,7 +858,7 @@ public: private: template void executeIntType( - ColumnsWithTypeAndName & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const + const ColumnsWithTypeAndName & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const { const ColumnAggregateFunction * column_ptrs[2]; bool is_column_const[2]; @@ -967,7 +967,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const IDataType * from_type = arguments[0].type.get(); const DataTypeAggregateFunction * aggr_type = typeid_cast(from_type); @@ -987,7 +987,7 @@ public: private: template - ColumnPtr executeBitmapData(ColumnsWithTypeAndName & arguments, size_t input_rows_count) const + ColumnPtr executeBitmapData(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { const ColumnAggregateFunction * column_ptrs[2]; bool is_column_const[2]; diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 6ae75318f72..ac3262f5131 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -88,7 +88,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const auto & col_type_name = arguments[0]; const ColumnPtr & column = col_type_name.column; @@ -168,7 +168,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const auto & col_type_name = arguments[0]; const ColumnPtr & column = col_type_name.column; @@ -277,7 +277,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnPtr & column = arguments[0].column; @@ -339,7 +339,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnPtr & column = arguments[0].column; @@ -407,7 +407,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnPtr & column = arguments[0].column; @@ -460,7 +460,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const auto & col_type_name = arguments[0]; const ColumnPtr & column = col_type_name.column; @@ -578,7 +578,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnPtr & column = arguments[0].column; @@ -688,7 +688,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnPtr & column = arguments[0].column; @@ -755,7 +755,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnWithTypeAndName & col_type_name = arguments[0]; const ColumnPtr & column = col_type_name.column; @@ -857,7 +857,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnWithTypeAndName & col_type_name = arguments[0]; const ColumnPtr & column = col_type_name.column; @@ -1187,7 +1187,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const IColumn * column = arguments[0].column.get(); ColumnPtr res_column; @@ -1255,7 +1255,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnPtr & column = arguments[0].column; @@ -1335,7 +1335,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { auto col_str = ColumnString::create(); ColumnString::Chars & out_vec = col_str->getChars(); @@ -1461,7 +1461,7 @@ public: } } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const IColumn * in_column = arguments[0].column.get(); ColumnPtr out_column; @@ -1599,7 +1599,7 @@ public: } } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const IColumn * column = arguments[0].column.get(); ColumnPtr res_column; @@ -1668,7 +1668,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const auto & col_type_name_ip = arguments[0]; const ColumnPtr & column_ip = col_type_name_ip.column; @@ -1782,7 +1782,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const auto & col_type_name_ip = arguments[0]; const ColumnPtr & column_ip = col_type_name_ip.column; diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 057f52501e5..e674f8690ff 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -1136,7 +1136,7 @@ public: return std::make_shared(); } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const auto & col_with_type_and_name_left = arguments[0]; const auto & col_with_type_and_name_right = arguments[1]; diff --git a/src/Functions/FunctionsConsistentHashing.h b/src/Functions/FunctionsConsistentHashing.h index edadfd659e2..faf66579fc4 100644 --- a/src/Functions/FunctionsConsistentHashing.h +++ b/src/Functions/FunctionsConsistentHashing.h @@ -65,7 +65,7 @@ public: return {1}; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { if (isColumnConst(*arguments[1].column)) return executeConstBuckets(arguments); @@ -93,7 +93,7 @@ private: return static_cast(buckets); } - ColumnPtr executeConstBuckets(ColumnsWithTypeAndName & arguments) const + ColumnPtr executeConstBuckets(const ColumnsWithTypeAndName & arguments) const { Field buckets_field = (*arguments[1].column)[0]; BucketsType num_buckets; diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 70e8904cfc1..6554c02b36a 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -102,7 +102,7 @@ struct ConvertImpl template static ColumnPtr NO_SANITIZE_UNDEFINED execute( - ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/, + const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/, Additions additions [[maybe_unused]] = Additions()) { const ColumnWithTypeAndName & named_from = arguments[0]; @@ -442,7 +442,7 @@ struct FormatImpl> template struct ConvertImpl, DataTypeNumber, Name> { - static ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) + static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) { return arguments[0].column; } @@ -455,7 +455,7 @@ struct ConvertImpl, ColumnDecimal, ColumnVector>; - static ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) + static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) { const auto & col_with_type_and_name = arguments[0]; const auto & type = static_cast(*col_with_type_and_name.type); @@ -509,7 +509,7 @@ struct ConvertImpl - static ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr & res_type, size_t input_rows_count, + static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & res_type, size_t input_rows_count, Additions additions [[maybe_unused]] = Additions()) { using ColVecTo = typename ToDataType::ColumnType; @@ -932,7 +932,7 @@ struct ConvertImpl template struct ConvertImpl, T, Name> { - static ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) + static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) { return arguments[0].column; } @@ -945,7 +945,7 @@ struct ConvertImpl, T, Name> template struct ConvertImpl { - static ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) + static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) { if (const ColumnFixedString * col_from = checkAndGetColumn(arguments[0].column.get())) { @@ -1141,7 +1141,7 @@ public: ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } bool canBeExecutedOnDefaultArguments() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { try { @@ -1186,7 +1186,7 @@ public: } private: - ColumnPtr executeInternal(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const + ColumnPtr executeInternal(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const { if (arguments.empty()) throw Exception{"Function " + getName() + " expects at least 1 arguments", @@ -1406,7 +1406,7 @@ public: } template - ColumnPtr executeInternal(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, UInt32 scale = 0) const + ColumnPtr executeInternal(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, UInt32 scale = 0) const { const IDataType * from_type = arguments[0].type.get(); @@ -1424,7 +1424,7 @@ public: return nullptr; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { ColumnPtr result_column; @@ -1880,7 +1880,7 @@ public: String getName() const override { return name; } protected: - ColumnPtr execute(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) override + ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { /// drop second argument, pass others ColumnsWithTypeAndName new_arguments{arguments.front()}; diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index 7c1221601f6..01456365740 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -183,7 +183,7 @@ public: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { /// The dictionary key that defines the "point of view". std::string dict_key; @@ -279,7 +279,7 @@ public: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { /// The dictionary key that defines the "point of view". std::string dict_key; @@ -415,7 +415,7 @@ public: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { /// The dictionary key that defines the "point of view". std::string dict_key; @@ -620,7 +620,7 @@ public: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { RegionsNames::Language language = RegionsNames::Language::ru; diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index 0fae3de1fb2..92a1389d212 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -163,7 +163,7 @@ private: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { /** Do not require existence of the dictionary if the function is called for empty columns. * This is needed to allow successful query analysis on a server, @@ -204,7 +204,7 @@ private: template ColumnPtr executeDispatchSimple( - ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -227,7 +227,7 @@ private: template ColumnPtr executeDispatchComplex( - ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -324,7 +324,7 @@ private: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (input_rows_count == 0) return result_type->createColumn(); @@ -359,7 +359,7 @@ private: template ColumnPtr executeDispatch( - ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -388,7 +388,7 @@ private: template ColumnPtr executeDispatchComplex( - ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -423,7 +423,7 @@ private: template ColumnPtr executeDispatchRange( - ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -502,7 +502,7 @@ private: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (input_rows_count == 0) return result_type->createColumn(); @@ -621,7 +621,7 @@ private: template ColumnPtr executeDispatchComplex( - ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -839,7 +839,7 @@ private: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (input_rows_count == 0) return result_type->createColumn(); @@ -873,7 +873,7 @@ private: } template - ColumnPtr executeDispatch(ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + ColumnPtr executeDispatch(const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -926,7 +926,7 @@ private: template ColumnPtr executeDispatchComplex( - ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -967,7 +967,7 @@ private: template ColumnPtr executeDispatchRange( - ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -1094,7 +1094,7 @@ private: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (input_rows_count == 0) return result_type->createColumn(); @@ -1127,7 +1127,7 @@ private: } template - ColumnPtr executeDispatch(ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + ColumnPtr executeDispatch(const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -1150,7 +1150,7 @@ private: template ColumnPtr executeDispatch( - ColumnsWithTypeAndName & arguments, const DictionaryType * dict, + const ColumnsWithTypeAndName & arguments, const DictionaryType * dict, const std::string & attr_name, const ColumnUInt64 * id_col) const { const auto * default_col_untyped = arguments[3].column.get(); @@ -1189,7 +1189,7 @@ private: template ColumnPtr executeDispatch( - ColumnsWithTypeAndName & arguments, const DictionaryType * dict, + const ColumnsWithTypeAndName & arguments, const DictionaryType * dict, const std::string & attr_name, const ColumnConst * id_col) const { const auto * default_col_untyped = arguments[3].column.get(); @@ -1246,7 +1246,7 @@ private: template ColumnPtr executeDispatchComplex( - ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -1472,7 +1472,7 @@ private: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { return impl->executeImpl(arguments, result_type, input_rows_count); } @@ -1613,7 +1613,7 @@ private: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { return impl->executeImpl(arguments, result_type, input_rows_count); } @@ -1661,7 +1661,7 @@ private: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (input_rows_count == 0) return result_type->createColumn(); @@ -1679,7 +1679,7 @@ private: } template - ColumnPtr executeDispatch(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const std::shared_ptr & dict_ptr) const + ColumnPtr executeDispatch(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -1814,7 +1814,7 @@ private: bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (input_rows_count == 0) return result_type->createColumn(); @@ -1832,7 +1832,7 @@ private: } template - ColumnPtr executeDispatch(ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const + ColumnPtr executeDispatch(const ColumnsWithTypeAndName & arguments, const std::shared_ptr & dict_ptr) const { const auto * dict = typeid_cast(dict_ptr.get()); if (!dict) diff --git a/src/Functions/FunctionsExternalModels.cpp b/src/Functions/FunctionsExternalModels.cpp index 9c1892012e1..ecec9383252 100644 --- a/src/Functions/FunctionsExternalModels.cpp +++ b/src/Functions/FunctionsExternalModels.cpp @@ -69,7 +69,7 @@ DataTypePtr FunctionModelEvaluate::getReturnTypeImpl(const ColumnsWithTypeAndNam return type; } -ColumnPtr FunctionModelEvaluate::executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const +ColumnPtr FunctionModelEvaluate::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const { const auto * name_col = checkAndGetColumnConst(arguments[0].column.get()); if (!name_col) @@ -85,7 +85,7 @@ ColumnPtr FunctionModelEvaluate::executeImpl(ColumnsWithTypeAndName & arguments, column_ptrs.reserve(arguments.size()); for (auto arg : ext::range(1, arguments.size())) { - auto & column = arguments[arg].column; + const auto & column = arguments[arg].column; column_ptrs.push_back(column.get()); if (auto full_column = column->convertToFullColumnIfConst()) { diff --git a/src/Functions/FunctionsExternalModels.h b/src/Functions/FunctionsExternalModels.h index 336dc164248..9bb6cc5a77c 100644 --- a/src/Functions/FunctionsExternalModels.h +++ b/src/Functions/FunctionsExternalModels.h @@ -32,7 +32,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override; - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override; private: const ExternalModelsLoader & models_loader; diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index 17165e12e37..fca27fe2f14 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -555,7 +555,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { if (const ColumnString * col_from = checkAndGetColumn(arguments[0].column.get())) { @@ -616,7 +616,7 @@ private: using ToType = typename Impl::ReturnType; template - ColumnPtr executeType(ColumnsWithTypeAndName & arguments) const + ColumnPtr executeType(const ColumnsWithTypeAndName & arguments) const { using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; @@ -659,7 +659,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const IDataType * from_type = arguments[0].type.get(); WhichDataType which(from_type); @@ -713,7 +713,7 @@ public: #endif } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { return selector.selectAndExecute(arguments, result_type, input_rows_count); } @@ -1065,7 +1065,7 @@ public: return std::make_shared>(); } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { size_t rows = input_rows_count; auto col_to = ColumnVector::create(rows); @@ -1107,7 +1107,7 @@ public: #endif } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { return selector.selectAndExecute(arguments, result_type, input_rows_count); } @@ -1230,7 +1230,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const auto arg_count = arguments.size(); @@ -1243,7 +1243,7 @@ public: } private: - ColumnPtr executeSingleArg(ColumnsWithTypeAndName & arguments) const + ColumnPtr executeSingleArg(const ColumnsWithTypeAndName & arguments) const { const auto * col_untyped = arguments.front().column.get(); @@ -1273,7 +1273,7 @@ private: " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; } - ColumnPtr executeTwoArgs(ColumnsWithTypeAndName & arguments) const + ColumnPtr executeTwoArgs(const ColumnsWithTypeAndName & arguments) const { const auto * level_col = arguments.back().column.get(); if (!isColumnConst(*level_col)) diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index 7478c1627af..7516600ac85 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -10,7 +10,7 @@ namespace ErrorCodes } -std::vector FunctionJSONHelpers::prepareMoves(const char * function_name, ColumnsWithTypeAndName & columns, size_t first_index_argument, size_t num_index_arguments) +std::vector FunctionJSONHelpers::prepareMoves(const char * function_name, const ColumnsWithTypeAndName & columns, size_t first_index_argument, size_t num_index_arguments) { std::vector moves; moves.reserve(num_index_arguments); diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 0fcf1f57f82..aea5829eaef 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -55,7 +55,7 @@ public: class Executor { public: - static ColumnPtr run(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) + static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) { MutableColumnPtr to{result_type->createColumn()}; to->reserve(input_rows_count); @@ -166,7 +166,7 @@ private: String key; }; - static std::vector prepareMoves(const char * function_name, ColumnsWithTypeAndName & columns, size_t first_index_argument, size_t num_index_arguments); + static std::vector prepareMoves(const char * function_name, const ColumnsWithTypeAndName & columns, size_t first_index_argument, size_t num_index_arguments); /// Performs moves of types MoveType::Index and MoveType::ConstIndex. template @@ -286,7 +286,7 @@ public: return Impl::getReturnType(Name::name, arguments); } - ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { /// Choose JSONParser. #if USE_SIMDJSON diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index 3e19516daaa..ab8e1cfc0b2 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -509,7 +509,7 @@ DataTypePtr FunctionAnyArityLogical::getReturnTypeImpl(const DataTyp template ColumnPtr FunctionAnyArityLogical::executeImpl( - ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const + const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const { ColumnRawPtrs args_in; for (const auto & arg_index : arguments) @@ -550,7 +550,7 @@ DataTypePtr FunctionUnaryLogical::getReturnTypeImpl(const DataTypes } template