From 58dd0cbe501fc0fd2fdb0991bed425b406a9ac20 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Wed, 24 Sep 2014 18:44:57 +0400 Subject: [PATCH 01/29] extend parser for default column declaration support. [#METR-12739] --- .../include/DB/Parsers/ASTColumnDeclaration.h | 44 +++++++++ dbms/include/DB/Parsers/ParserCreateQuery.h | 93 +++++++++++++++++++ dbms/src/Parsers/ParserCreateQuery.cpp | 7 +- 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 dbms/include/DB/Parsers/ASTColumnDeclaration.h diff --git a/dbms/include/DB/Parsers/ASTColumnDeclaration.h b/dbms/include/DB/Parsers/ASTColumnDeclaration.h new file mode 100644 index 00000000000..a4f017586d6 --- /dev/null +++ b/dbms/include/DB/Parsers/ASTColumnDeclaration.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +namespace DB +{ + +/** Name, type, default-specifier, default-expression. + * The type is optional if default-expression is specified. + */ +class ASTColumnDeclaration : public IAST +{ +public: + String name; + ASTPtr type; + String default_specifier; + ASTPtr default_expression; + + ASTColumnDeclaration() = default; + ASTColumnDeclaration(StringRange range) : IAST{range} {} + + String getID() const { return "ColumnDeclaration_" + name; } + + ASTPtr clone() const + { + const auto res = new ASTColumnDeclaration{*this}; + res->children.clear(); + + if (type) { + res->type = type->clone(); + res->children.push_back(res->type); + } + + if (default_expression) { + res->default_expression = default_expression->clone(); + res->children.push_back(res->default_expression); + } + + return res; + } +}; + +} + diff --git a/dbms/include/DB/Parsers/ParserCreateQuery.h b/dbms/include/DB/Parsers/ParserCreateQuery.h index 991d9c44f1d..f3492549ead 100644 --- a/dbms/include/DB/Parsers/ParserCreateQuery.h +++ b/dbms/include/DB/Parsers/ParserCreateQuery.h @@ -93,6 +93,99 @@ protected: }; +template +class IParserColumnDeclaration : public IParserBase +{ +protected: + const char * getName() const { return "column declaration"; } + bool parseImpl(Pos & pos, Pos end, ASTPtr & node, Expected & expected); +}; + +typedef IParserColumnDeclaration ParserColumnDeclaration; +typedef IParserColumnDeclaration ParserCompoundColumnDeclaration; + +template +bool IParserColumnDeclaration::parseImpl(Pos & pos, Pos end, ASTPtr & node, Expected & expected) +{ + NameParser name_parser; + ParserIdentifierWithOptionalParameters type_parser; + ParserWhiteSpaceOrComments ws; + ParserString s_default{"DEFAULT"}; + ParserString s_materialized{"MATERIALIZED"}; + ParserString s_alias{"ALIAS"}; + ParserExpressionElement expr_parser; + + const auto begin = pos; + + /// mandatory column name + ASTPtr name; + if (!name_parser.parse(pos, end, name, expected)) + return false; + + ws.ignore(pos, end, expected); + + /** column name should be followed by type name if it + * is not immediately followed by {DEFAULT, MATERIALIZED, ALIAS} */ + ASTPtr type; + const auto fallback_pos = pos; + if (!s_default.check(pos, end, expected) && + !s_materialized.check(pos, end, expected) && + !s_alias.check(pos, end, expected)) + { + /// reject sole column name without type + if (type_parser.parse(pos, end, type, expected)) + ws.ignore(pos, end, expected); + } else pos = fallback_pos; + + /// parse {DEFAULT, MATERIALIZED, ALIAS} + String default_specifier; + ASTPtr default_expression; + const auto pos_before_specifier = pos; + if (s_default.ignore(pos, end, expected) || + s_materialized.ignore(pos, end, expected) || + s_alias.ignore(pos, end, expected)) + { + default_specifier.assign(pos_before_specifier, pos); + + /// should be followed by an expression + ws.ignore(pos, end, expected); + + if (!expr_parser.parse(pos, end, default_expression, expected)) + { + pos = begin; + return false; + } + } + else if (!type) + { + pos = begin; + return false; + } + + /// @todo remove this + if (!type) { + const auto no_type = "NO_TYPE"; + auto pos = no_type; + type_parser.parse(pos, no_type + std::strlen(no_type), type, expected); + } + + const auto name_type_pair = new ASTNameTypePair{StringRange{begin, pos}}; + node = name_type_pair; + name_type_pair->name = typeid_cast(*name).name; + name_type_pair->type = type; + name_type_pair->children.push_back(std::move(type)); + + return true; +} + +class ParserColumnDeclarationList : public IParserBase +{ +protected: + const char * getName() const { return "column declaration list"; } + bool parseImpl(Pos & pos, Pos end, ASTPtr & node, Expected & expected); +}; + + /** ENGINE = name. */ class ParserEngine : public IParserBase { diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index 603588481d2..1ab39750b82 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -103,6 +103,11 @@ bool ParserNameTypePairList::parseImpl(Pos & pos, Pos end, ASTPtr & node, Expect return ParserList(ParserPtr(new ParserNameTypePair), ParserPtr(new ParserString(",")), false).parse(pos, end, node, expected); } +bool ParserColumnDeclarationList::parseImpl(Pos & pos, Pos end, ASTPtr & node, Expected & expected) +{ + return ParserList{ParserPtr{new ParserColumnDeclaration}, ParserPtr{new ParserString{","}}, false}.parse(pos, end, node, expected); +} + bool ParserEngine::parseImpl(Pos & pos, Pos end, ASTPtr & storage, Expected & expected) { @@ -155,7 +160,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, Expected & ParserString s_populate("POPULATE", true, true); ParserEngine engine_p; ParserIdentifier name_p; - ParserNameTypePairList columns_p; + ParserColumnDeclarationList columns_p; ASTPtr database; ASTPtr table; From 142d0a49c051aa3a7048b9a1d242d3bf945d498d Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 25 Sep 2014 17:40:26 +0400 Subject: [PATCH 02/29] add ASTColumnDeclaration, corresponding formatAST, modify Interpreters. [#METR-12739] --- dbms/include/DB/Parsers/ParserCreateQuery.h | 30 +++++--- dbms/include/DB/Parsers/formatAST.h | 2 + .../Interpreters/InterpreterCreateQuery.cpp | 75 +++++++++++++------ dbms/src/Parsers/formatAST.cpp | 27 ++++++- 4 files changed, 99 insertions(+), 35 deletions(-) diff --git a/dbms/include/DB/Parsers/ParserCreateQuery.h b/dbms/include/DB/Parsers/ParserCreateQuery.h index f3492549ead..439d4bc45e1 100644 --- a/dbms/include/DB/Parsers/ParserCreateQuery.h +++ b/dbms/include/DB/Parsers/ParserCreateQuery.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -132,10 +133,11 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, Pos end, ASTPtr !s_materialized.check(pos, end, expected) && !s_alias.check(pos, end, expected)) { - /// reject sole column name without type if (type_parser.parse(pos, end, type, expected)) ws.ignore(pos, end, expected); - } else pos = fallback_pos; + } + else + pos = fallback_pos; /// parse {DEFAULT, MATERIALIZED, ALIAS} String default_specifier; @@ -158,22 +160,26 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, Pos end, ASTPtr } else if (!type) { + /// reject sole column name without type pos = begin; return false; } - /// @todo remove this - if (!type) { - const auto no_type = "NO_TYPE"; - auto pos = no_type; - type_parser.parse(pos, no_type + std::strlen(no_type), type, expected); + const auto column_declaration = new ASTColumnDeclaration{StringRange{begin, pos}}; + node = column_declaration; + column_declaration->name = typeid_cast(*name).name; + if (type) + { + column_declaration->type = type; + column_declaration->children.push_back(std::move(type)); } - const auto name_type_pair = new ASTNameTypePair{StringRange{begin, pos}}; - node = name_type_pair; - name_type_pair->name = typeid_cast(*name).name; - name_type_pair->type = type; - name_type_pair->children.push_back(std::move(type)); + if (default_expression) + { + column_declaration->default_specifier = default_specifier; + column_declaration->default_expression = default_expression; + column_declaration->children.push_back(std::move(default_expression)); + } return true; } diff --git a/dbms/include/DB/Parsers/formatAST.h b/dbms/include/DB/Parsers/formatAST.h index a9be893e06c..f9f903037ca 100644 --- a/dbms/include/DB/Parsers/formatAST.h +++ b/dbms/include/DB/Parsers/formatAST.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -53,6 +54,7 @@ void formatAST(const ASTFunction & ast, std::ostream & s, size_t indent = 0, b void formatAST(const ASTIdentifier & ast, std::ostream & s, size_t indent = 0, bool hilite = true, bool one_line = false, bool need_parens = false); void formatAST(const ASTLiteral & ast, std::ostream & s, size_t indent = 0, bool hilite = true, bool one_line = false, bool need_parens = false); void formatAST(const ASTNameTypePair & ast, std::ostream & s, size_t indent = 0, bool hilite = true, bool one_line = false, bool need_parens = false); +void formatAST(const ASTColumnDeclaration & ast, std::ostream & s, size_t indent = 0, bool hilite = true, bool one_line = false, bool need_parens = false); void formatAST(const ASTAsterisk & ast, std::ostream & s, size_t indent = 0, bool hilite = true, bool one_line = false, bool need_parens = false); void formatAST(const ASTOrderByElement & ast, std::ostream & s, size_t indent = 0, bool hilite = true, bool one_line = false, bool need_parens = false); void formatAST(const ASTSubquery & ast, std::ostream & s, size_t indent = 0, bool hilite = true, bool one_line = false, bool need_parens = false); diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 6b392173859..5b83477a7c7 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -236,41 +237,71 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) NamesAndTypesList InterpreterCreateQuery::parseColumns(ASTPtr expression_list, const DataTypeFactory & data_type_factory) { NamesAndTypesList columns; - ASTExpressionList & columns_list = typeid_cast(*expression_list); - for (const ASTPtr & ast : columns_list.children) + auto & columns_list = typeid_cast(*expression_list); + + /// List of columns requiring type-deduction or default_expression type-check + using postprocess_column = std::pair; + std::vector postprocess_columns{}; + + for (const auto & ast : columns_list.children) { - const ASTNameTypePair & name_and_type_pair = typeid_cast(*ast); - StringRange type_range = name_and_type_pair.type->range; - columns.push_back(NameAndTypePair( - name_and_type_pair.name, - data_type_factory.get(String(type_range.first, type_range.second - type_range.first)))); + const auto & col_decl = typeid_cast(*ast); + /// deduce type from default_expression if no type specified + if (col_decl.type) + { + const auto & type_range = col_decl.type->range; + columns.emplace_back( + col_decl.name, + data_type_factory.get({ type_range.first, type_range.second })); + } + else + columns.emplace_back(col_decl.name, nullptr); + + /// add column to postprocess if there is either no type or a default_expression specified + if (!col_decl.type || col_decl.default_expression) + { + postprocess_columns.emplace_back(&columns.back(), &col_decl); + } } - columns = *DataTypeNested::expandNestedColumns(columns); - return columns; + + /// deduce type or wrap default_expression in conversion-function if necessary + for (auto & column : postprocess_columns) + { + const auto name_and_type_ptr = column.first; + const auto col_decl_ptr = column.second; + + /// @todo deduce real type + + if (!col_decl_ptr->type) + name_and_type_ptr->type = data_type_factory.get("UInt32"); + } + + return *DataTypeNested::expandNestedColumns(columns); } ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns) { - ASTPtr columns_list_ptr = new ASTExpressionList; + ASTPtr columns_list_ptr{new ASTExpressionList}; ASTExpressionList & columns_list = typeid_cast(*columns_list_ptr); - for (const NameAndTypePair & it : columns) + for (const auto & column : columns) { - ASTPtr name_and_type_pair_ptr = new ASTNameTypePair; - ASTNameTypePair & name_and_type_pair = typeid_cast(*name_and_type_pair_ptr); - name_and_type_pair.name = it.name; - StringPtr type_name = new String(it.type->getName()); + const auto column_declaration = new ASTColumnDeclaration; + ASTPtr column_declaration_ptr{column_declaration}; + + column_declaration->name = column.name; + + StringPtr type_name{new String(column.type->getName())}; + auto pos = type_name->data(); + const auto end = pos + type_name->size(); ParserIdentifierWithOptionalParameters storage_p; - Expected expected = ""; - const char * pos = type_name->data(); - const char * end = pos + type_name->size(); - - if (!storage_p.parse(pos, end, name_and_type_pair.type, expected)) + Expected expected{""}; + if (!storage_p.parse(pos, end, column_declaration->type, expected)) throw Exception("Cannot parse data type.", ErrorCodes::SYNTAX_ERROR); - name_and_type_pair.type->query_string = type_name; - columns_list.children.push_back(name_and_type_pair_ptr); + column_declaration->type->query_string = type_name; + columns_list.children.push_back(column_declaration_ptr); } return columns_list_ptr; diff --git a/dbms/src/Parsers/formatAST.cpp b/dbms/src/Parsers/formatAST.cpp index 7b696b49510..f1692c25dab 100644 --- a/dbms/src/Parsers/formatAST.cpp +++ b/dbms/src/Parsers/formatAST.cpp @@ -39,6 +39,12 @@ String backQuoteIfNeed(const String & x) } +String hightlight(const String & keyword, const String & color_sequence, const bool hilite) +{ + return hilite ? color_sequence + keyword + hilite_none : keyword; +} + + void formatAST(const IAST & ast, std::ostream & s, size_t indent, bool hilite, bool one_line, bool need_parens) { @@ -64,6 +70,7 @@ void formatAST(const IAST & ast, std::ostream & s, size_t indent, bool hilite, b DISPATCH(Identifier) DISPATCH(Literal) DISPATCH(NameTypePair) + DISPATCH(ColumnDeclaration) DISPATCH(Asterisk) DISPATCH(OrderByElement) DISPATCH(Subquery) @@ -681,6 +688,25 @@ void formatAST(const ASTNameTypePair & ast, std::ostream & s, size_t indent, bo formatAST(*ast.type, s, indent, hilite, one_line); } +void formatAST(const ASTColumnDeclaration & ast, std::ostream & s, size_t indent, bool hilite, bool one_line, bool need_parens) +{ + std::string indent_str = one_line ? "" : std::string(4 * indent, ' '); + std::string nl_or_ws = one_line ? " " : "\n"; + + s << nl_or_ws << indent_str << backQuoteIfNeed(ast.name); + if (ast.type) + { + s << ' '; + formatAST(*ast.type, s, indent, hilite, one_line); + } + + if (ast.default_expression) + { + s << ' ' << hightlight(ast.default_specifier, hilite_keyword, hilite) << ' '; + formatAST(*ast.default_expression, s, indent, hilite, one_line); + } +} + void formatAST(const ASTAsterisk & ast, std::ostream & s, size_t indent, bool hilite, bool one_line, bool need_parens) { s << "*"; @@ -835,4 +861,3 @@ String formatColumnsForCreateQuery(NamesAndTypesList & columns) } } - From cee035439e12d6f82944861e17ac2ac3d26592f1 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 25 Sep 2014 19:00:12 +0400 Subject: [PATCH 03/29] add makeASTFunction factory-function. #[NETR-12739] --- dbms/include/DB/Parsers/ASTFunction.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dbms/include/DB/Parsers/ASTFunction.h b/dbms/include/DB/Parsers/ASTFunction.h index 37ea10fb06a..238efc87644 100644 --- a/dbms/include/DB/Parsers/ASTFunction.h +++ b/dbms/include/DB/Parsers/ASTFunction.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -81,4 +82,20 @@ public: } }; + +template +ASTPtr makeASTFunction(const String & name, Args &&... args) +{ + const auto function = new ASTFunction{}; + ASTPtr result{function}; + + function->name = name; + function->arguments = new ASTExpressionList{}; + function->children.push_back(function->arguments); + + function->arguments->children = { std::forward(args)... }; + + return result; +} + } From 1d04061a0ec1e69d2be330f3c9e384c0ff7edf54 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 25 Sep 2014 19:01:09 +0400 Subject: [PATCH 04/29] implement type-deduction and type-enforcing. [#METR-12739] --- .../DB/Interpreters/InterpreterCreateQuery.h | 13 +++--- .../Interpreters/InterpreterCreateQuery.cpp | 43 ++++++++++++++++--- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/dbms/include/DB/Interpreters/InterpreterCreateQuery.h b/dbms/include/DB/Interpreters/InterpreterCreateQuery.h index ddcf7672e48..c7e1405de51 100644 --- a/dbms/include/DB/Interpreters/InterpreterCreateQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterCreateQuery.h @@ -14,7 +14,7 @@ class InterpreterCreateQuery { public: InterpreterCreateQuery(ASTPtr query_ptr_, Context & context_); - + /** В случае таблицы: добавляет созданную таблицу в контекст, а также возвращает её. * В случае БД: добавляет созданную БД в контекст и возвращает NULL. * assume_metadata_exists - не проверять наличие файла с метаданными и не создавать его @@ -22,12 +22,15 @@ public: */ StoragePtr execute(bool assume_metadata_exists = false); - /** AST в список столбцов с типами и обратно. Столбцы типа Nested развернуты в список настоящих столбцов. - */ - static NamesAndTypesList parseColumns(ASTPtr expression_list, const DataTypeFactory & data_type_factory); + /// Список столбцов с типами в AST. static ASTPtr formatColumns(const NamesAndTypesList & columns); - + private: + /// AST в список столбцов с типами. Столбцы типа Nested развернуты в список настоящих столбцов. + NamesAndTypesList parseColumns(ASTPtr expression_list, const DataTypeFactory & data_type_factory); + + DataTypePtr deduceType(const ASTPtr & expr, const NamesAndTypesList & columns) const; + ASTPtr query_ptr; Context context; }; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 5b83477a7c7..d9ae9c200a1 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -21,6 +21,7 @@ #include #include +#include #include @@ -236,16 +237,20 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) NamesAndTypesList InterpreterCreateQuery::parseColumns(ASTPtr expression_list, const DataTypeFactory & data_type_factory) { - NamesAndTypesList columns; + /// list of table columns in correct order + NamesAndTypesList columns{}; + /// list of processed columns for type-deduction + NamesAndTypesList processed_columns{}; + auto & columns_list = typeid_cast(*expression_list); /// List of columns requiring type-deduction or default_expression type-check - using postprocess_column = std::pair; + using postprocess_column = std::pair; std::vector postprocess_columns{}; - for (const auto & ast : columns_list.children) + for (auto & ast : columns_list.children) { - const auto & col_decl = typeid_cast(*ast); + auto & col_decl = typeid_cast(*ast); /// deduce type from default_expression if no type specified if (col_decl.type) { @@ -253,6 +258,8 @@ NamesAndTypesList InterpreterCreateQuery::parseColumns(ASTPtr expression_list, c columns.emplace_back( col_decl.name, data_type_factory.get({ type_range.first, type_range.second })); + + processed_columns.emplace_back(columns.back()); } else columns.emplace_back(col_decl.name, nullptr); @@ -264,16 +271,31 @@ NamesAndTypesList InterpreterCreateQuery::parseColumns(ASTPtr expression_list, c } } + /// @todo check for presence of cycles in default_expressions, throw if detected /// deduce type or wrap default_expression in conversion-function if necessary for (auto & column : postprocess_columns) { const auto name_and_type_ptr = column.first; const auto col_decl_ptr = column.second; - /// @todo deduce real type + /// deduce real type + auto type = deduceType(col_decl_ptr->default_expression, processed_columns); - if (!col_decl_ptr->type) - name_and_type_ptr->type = data_type_factory.get("UInt32"); + if (!name_and_type_ptr->type) + name_and_type_ptr->type = type; + else if (typeid(*name_and_type_ptr->type) != typeid(*type)) + { + /// wrap default_expression in a type-conversion function + col_decl_ptr->default_expression = makeASTFunction( + "to" + name_and_type_ptr->type->getName(), + col_decl_ptr->default_expression); + + col_decl_ptr->children.clear(); + col_decl_ptr->children.push_back(col_decl_ptr->type); + col_decl_ptr->children.push_back(col_decl_ptr->default_expression); + } + + processed_columns.emplace_back(*name_and_type_ptr); } return *DataTypeNested::expandNestedColumns(columns); @@ -307,4 +329,11 @@ ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns) return columns_list_ptr; } +DataTypePtr InterpreterCreateQuery::deduceType(const ASTPtr & expr, const NamesAndTypesList & columns) const +{ + const auto actions = ExpressionAnalyzer{expr, context, columns}.getActions(false); + + return actions->getSampleBlock().getByName(expr->getColumnName()).type; +} + } From 529674ce6ad9fb62009b7e5612df5e22e05daa7b Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Mon, 29 Sep 2014 17:32:16 +0400 Subject: [PATCH 05/29] detect cycles, deduce and check types. [#METR-12739] --- dbms/include/DB/Parsers/ASTFunction.h | 17 ++++ dbms/include/DB/Parsers/ParserCreateQuery.h | 15 ++- .../Interpreters/InterpreterCreateQuery.cpp | 94 ++++++++++++------- dbms/src/Parsers/ExpressionListParsers.cpp | 10 +- 4 files changed, 87 insertions(+), 49 deletions(-) diff --git a/dbms/include/DB/Parsers/ASTFunction.h b/dbms/include/DB/Parsers/ASTFunction.h index 238efc87644..2b31a2fada2 100644 --- a/dbms/include/DB/Parsers/ASTFunction.h +++ b/dbms/include/DB/Parsers/ASTFunction.h @@ -98,4 +98,21 @@ ASTPtr makeASTFunction(const String & name, Args &&... args) return result; } + +template +ASTPtr makeASTFunction(const String & name, const StringRange & function_range, + const StringRange & arguments_range, Args &&... args) +{ + const auto function = new ASTFunction{function_range}; + ASTPtr result{function}; + + function->name = name; + function->arguments = new ASTExpressionList{arguments_range}; + function->children.push_back(function->arguments); + + function->arguments->children = { std::forward(args)... }; + + return result; +} + } diff --git a/dbms/include/DB/Parsers/ParserCreateQuery.h b/dbms/include/DB/Parsers/ParserCreateQuery.h index 439d4bc45e1..97ff2c60ef8 100644 --- a/dbms/include/DB/Parsers/ParserCreateQuery.h +++ b/dbms/include/DB/Parsers/ParserCreateQuery.h @@ -117,6 +117,10 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, Pos end, ASTPtr ParserExpressionElement expr_parser; const auto begin = pos; + const auto reset_pos_and_return = [&pos, begin] { + pos = begin; + return false; + }; /// mandatory column name ASTPtr name; @@ -153,17 +157,10 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, Pos end, ASTPtr ws.ignore(pos, end, expected); if (!expr_parser.parse(pos, end, default_expression, expected)) - { - pos = begin; - return false; - } + return reset_pos_and_return(); } else if (!type) - { - /// reject sole column name without type - pos = begin; - return false; - } + return reset_pos_and_return(); /// reject sole column name without type const auto column_declaration = new ASTColumnDeclaration{StringRange{begin, pos}}; node = column_declaration; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index d9ae9c200a1..bfd44f81938 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -239,63 +239,93 @@ NamesAndTypesList InterpreterCreateQuery::parseColumns(ASTPtr expression_list, c { /// list of table columns in correct order NamesAndTypesList columns{}; - /// list of processed columns for type-deduction - NamesAndTypesList processed_columns{}; auto & columns_list = typeid_cast(*expression_list); - /// List of columns requiring type-deduction or default_expression type-check using postprocess_column = std::pair; - std::vector postprocess_columns{}; + /// Columns requiring type-deduction or default_expression type-check + std::vector defaulted_columns{}; + + /** all default_expressions as a single expression list, + * mixed with conversion-columns for each explicitly specified type */ + ASTPtr default_expr_list{new ASTExpressionList}; + default_expr_list->children.reserve(columns_list.children.size()); + + /// helper for setting aliases and chaining result to other functions + const auto set_alias = [] (ASTPtr ast, const String & alias) { + dynamic_cast(*ast).alias = alias; + + return ast; + }; for (auto & ast : columns_list.children) { auto & col_decl = typeid_cast(*ast); - /// deduce type from default_expression if no type specified + if (col_decl.type) { const auto & type_range = col_decl.type->range; - columns.emplace_back( - col_decl.name, + columns.emplace_back(col_decl.name, data_type_factory.get({ type_range.first, type_range.second })); - - processed_columns.emplace_back(columns.back()); } else columns.emplace_back(col_decl.name, nullptr); - /// add column to postprocess if there is either no type or a default_expression specified - if (!col_decl.type || col_decl.default_expression) + /// add column to postprocessing if there is a default_expression specified + if (col_decl.default_expression) { - postprocess_columns.emplace_back(&columns.back(), &col_decl); + defaulted_columns.emplace_back(&columns.back(), &col_decl); + + /** for columns with explicitly-specified type create two expressions: + * 1. default_expression aliased as column name with _tmp suffix + * 2. conversion of expression (1) to explicitly-specified type alias as column name */ + if (col_decl.type) + { + const auto tmp_column_name = col_decl.name + "_tmp"; + const auto & final_column_name = col_decl.name; + const auto conversion_function_name = "to" + columns.back().type->getName(); + + default_expr_list->children.emplace_back(set_alias( + makeASTFunction(conversion_function_name, ASTPtr{new ASTIdentifier{{}, tmp_column_name}}), + final_column_name)); + + default_expr_list->children.emplace_back(set_alias(col_decl.default_expression->clone(), tmp_column_name)); + } + else + default_expr_list->children.emplace_back(set_alias(col_decl.default_expression->clone(), col_decl.name)); } } - /// @todo check for presence of cycles in default_expressions, throw if detected - /// deduce type or wrap default_expression in conversion-function if necessary - for (auto & column : postprocess_columns) + /// set missing types and wrap default_expression's in a conversion-function if necessary + if (!defaulted_columns.empty()) { - const auto name_and_type_ptr = column.first; - const auto col_decl_ptr = column.second; + const auto actions = ExpressionAnalyzer{default_expr_list, context, columns}.getActions(true); + const auto block = actions->getSampleBlock(); - /// deduce real type - auto type = deduceType(col_decl_ptr->default_expression, processed_columns); - - if (!name_and_type_ptr->type) - name_and_type_ptr->type = type; - else if (typeid(*name_and_type_ptr->type) != typeid(*type)) + for (auto & column : defaulted_columns) { - /// wrap default_expression in a type-conversion function - col_decl_ptr->default_expression = makeASTFunction( - "to" + name_and_type_ptr->type->getName(), - col_decl_ptr->default_expression); + const auto name_and_type_ptr = column.first; + const auto col_decl_ptr = column.second; - col_decl_ptr->children.clear(); - col_decl_ptr->children.push_back(col_decl_ptr->type); - col_decl_ptr->children.push_back(col_decl_ptr->default_expression); + if (name_and_type_ptr->type) + { + const auto & tmp_column = block.getByName(col_decl_ptr->name + "_tmp"); + + /// type mismatch between explicitly specified and deduced type, add conversion + if (typeid(*name_and_type_ptr->type) != typeid(*tmp_column.type)) + { + col_decl_ptr->default_expression = makeASTFunction( + "to" + name_and_type_ptr->type->getName(), + col_decl_ptr->default_expression); + + col_decl_ptr->children.clear(); + col_decl_ptr->children.push_back(col_decl_ptr->type); + col_decl_ptr->children.push_back(col_decl_ptr->default_expression); + } + } + else + name_and_type_ptr->type = block.getByName(name_and_type_ptr->name).type; } - - processed_columns.emplace_back(*name_and_type_ptr); } return *DataTypeNested::expandNestedColumns(columns); diff --git a/dbms/src/Parsers/ExpressionListParsers.cpp b/dbms/src/Parsers/ExpressionListParsers.cpp index b4226b57075..9154aa8276b 100644 --- a/dbms/src/Parsers/ExpressionListParsers.cpp +++ b/dbms/src/Parsers/ExpressionListParsers.cpp @@ -210,14 +210,8 @@ bool ParserVariableArityOperatorList::parseImpl(Pos & pos, Pos end, ASTPtr & nod if (!arguments) { - ASTFunction * function = new ASTFunction; - ASTPtr function_node = function; - arguments = new ASTExpressionList; - function->arguments = arguments; - function->children.push_back(arguments); - function->name = function_name; - arguments->children.push_back(node); - node = function_node; + node = makeASTFunction(function_name, node); + arguments = static_cast(*node).arguments; } ASTPtr elem; From 3060ba9571557bdb7c07499580f6c2ee2b749ae4 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Mon, 29 Sep 2014 18:58:48 +0400 Subject: [PATCH 06/29] return ColumnDefaults from InterpreterCreateQuery::parseColumns. [#METR-12739] --- .../DB/Interpreters/InterpreterCreateQuery.h | 4 +- dbms/include/DB/Storages/ColumnDefault.h | 60 +++++++++++++++++++ .../Interpreters/InterpreterCreateQuery.cpp | 22 ++++++- 3 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 dbms/include/DB/Storages/ColumnDefault.h diff --git a/dbms/include/DB/Interpreters/InterpreterCreateQuery.h b/dbms/include/DB/Interpreters/InterpreterCreateQuery.h index c7e1405de51..382efa912fa 100644 --- a/dbms/include/DB/Interpreters/InterpreterCreateQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterCreateQuery.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB @@ -27,7 +28,8 @@ public: private: /// AST в список столбцов с типами. Столбцы типа Nested развернуты в список настоящих столбцов. - NamesAndTypesList parseColumns(ASTPtr expression_list, const DataTypeFactory & data_type_factory); + using ColumnsAndDefaults = std::pair; + ColumnsAndDefaults parseColumns(ASTPtr expression_list, const DataTypeFactory & data_type_factory); DataTypePtr deduceType(const ASTPtr & expr, const NamesAndTypesList & columns) const; diff --git a/dbms/include/DB/Storages/ColumnDefault.h b/dbms/include/DB/Storages/ColumnDefault.h new file mode 100644 index 00000000000..c6dd3f986c7 --- /dev/null +++ b/dbms/include/DB/Storages/ColumnDefault.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +namespace DB +{ + enum struct ColumnDefaultType + { + Default, + Materialized, + Alias + }; +} + +namespace std +{ + template<> struct hash + { + size_t operator()(const DB::ColumnDefaultType type) const + { + return hash{}(static_cast(type)); + } + }; +} + +namespace DB +{ + inline ColumnDefaultType columnDefaultTypeFromString(const String& str) + { + static const std::unordered_map map{ + { "DEFAULT", ColumnDefaultType::Default }, + { "MATERIALIZED", ColumnDefaultType::Materialized }, + { "ALIAS", ColumnDefaultType::Alias } + }; + + const auto it = map.find(str); + return it != std::end(map) ? it->second : throw Exception{"Unknown column default specifier: " + str}; + } + + inline String toString(const ColumnDefaultType type) + { + static const std::unordered_map map{ + { ColumnDefaultType::Default, "DEFAULT" }, + { ColumnDefaultType::Materialized, "MATERIALIZED" }, + { ColumnDefaultType::Alias, "ALIAS" } + }; + + const auto it = map.find(type); + return it != std::end(map) ? it->second : throw Exception{"Invalid ColumnDefaultType"}; + } + + struct ColumnDefault + { + ColumnDefaultType type; + ASTPtr expression; + }; + + using ColumnDefaults = std::unordered_map; +} diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index bfd44f81938..f9518dfa0e1 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -89,6 +89,7 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) StoragePtr res; String storage_name; NamesAndTypesListPtr columns = new NamesAndTypesList; + ColumnDefaults column_defaults{}; StoragePtr as_storage; IStorage::TableStructureReadLockPtr as_storage_lock; @@ -117,7 +118,9 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) /// Получаем список столбцов if (create.columns) { - columns = new NamesAndTypesList(parseColumns(create.columns, context.getDataTypeFactory())); + auto && columns_and_defaults = parseColumns(create.columns, context.getDataTypeFactory()); + columns = new NamesAndTypesList{std::move(columns_and_defaults.first)}; + column_defaults = std::move(columns_and_defaults.second); } else if (!create.as_table.empty()) columns = new NamesAndTypesList(as_storage->getColumnsList()); @@ -235,10 +238,12 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) return res; } -NamesAndTypesList InterpreterCreateQuery::parseColumns(ASTPtr expression_list, const DataTypeFactory & data_type_factory) +InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( + ASTPtr expression_list, const DataTypeFactory & data_type_factory) { /// list of table columns in correct order NamesAndTypesList columns{}; + ColumnDefaults defaults{}; auto & columns_list = typeid_cast(*expression_list); @@ -292,7 +297,13 @@ NamesAndTypesList InterpreterCreateQuery::parseColumns(ASTPtr expression_list, c default_expr_list->children.emplace_back(set_alias(col_decl.default_expression->clone(), tmp_column_name)); } else + { default_expr_list->children.emplace_back(set_alias(col_decl.default_expression->clone(), col_decl.name)); + defaults.emplace(col_decl.name, ColumnDefault{ + columnDefaultTypeFromString(col_decl.default_specifier), + col_decl.default_expression + }); + } } } @@ -325,10 +336,15 @@ NamesAndTypesList InterpreterCreateQuery::parseColumns(ASTPtr expression_list, c } else name_and_type_ptr->type = block.getByName(name_and_type_ptr->name).type; + + defaults.emplace(col_decl_ptr->name, ColumnDefault{ + columnDefaultTypeFromString(col_decl_ptr->default_specifier), + col_decl_ptr->default_expression + }); } } - return *DataTypeNested::expandNestedColumns(columns); + return { *DataTypeNested::expandNestedColumns(columns), defaults }; } ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns) From 805202021822781b22d11bed1e47e79b95b8ae4b Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 30 Sep 2014 07:08:47 +0400 Subject: [PATCH 07/29] add draft version of DEFAULT, MATERIALIZED/ALIAS currently parsed but not taken into account. [#METR-12739] --- dbms/include/DB/Core/Block.h | 6 + .../AddingDefaultBlockInputStream.h | 12 +- .../AddingDefaultBlockOutputStream.h | 13 ++- dbms/include/DB/Functions/FunctionsString.h | 18 +-- .../DB/Interpreters/InterpreterCreateQuery.h | 9 +- dbms/include/DB/Parsers/ASTWithAlias.h | 8 ++ dbms/include/DB/Parsers/ParserCreateQuery.h | 3 +- dbms/include/DB/Storages/IStorage.h | 8 +- dbms/include/DB/Storages/ITableDeclaration.h | 14 ++- .../DB/Storages/MergeTree/MergeTreeData.h | 6 +- dbms/include/DB/Storages/StorageChunkMerger.h | 28 +++-- dbms/include/DB/Storages/StorageChunkRef.h | 16 +-- dbms/include/DB/Storages/StorageChunks.h | 50 ++++---- dbms/include/DB/Storages/StorageDistributed.h | 14 +++ dbms/include/DB/Storages/StorageFactory.h | 2 + dbms/include/DB/Storages/StorageLog.h | 42 ++++--- .../DB/Storages/StorageMaterializedView.h | 23 +++- dbms/include/DB/Storages/StorageMemory.h | 25 +++- dbms/include/DB/Storages/StorageMerge.h | 22 +++- dbms/include/DB/Storages/StorageMergeTree.h | 36 +++--- dbms/include/DB/Storages/StorageNull.h | 16 ++- .../DB/Storages/StorageReplicatedMergeTree.h | 8 +- dbms/include/DB/Storages/StorageTinyLog.h | 28 +++-- dbms/include/DB/Storages/StorageView.h | 20 +++- .../DB/TableFunctions/TableFunctionMerge.h | 3 +- dbms/src/Core/Block.cpp | 60 ++++++++-- .../Interpreters/InterpreterCreateQuery.cpp | 109 +++++++++++++----- .../Interpreters/InterpreterInsertQuery.cpp | 10 +- dbms/src/Storages/ITableDeclaration.cpp | 27 ++++- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 4 +- dbms/src/Storages/StorageChunkMerger.cpp | 26 +++-- dbms/src/Storages/StorageChunks.cpp | 38 +++--- dbms/src/Storages/StorageDistributed.cpp | 31 ++++- dbms/src/Storages/StorageFactory.cpp | 58 +++++++--- dbms/src/Storages/StorageLog.cpp | 26 ++++- dbms/src/Storages/StorageMaterializedView.cpp | 30 ++++- dbms/src/Storages/StorageMemory.cpp | 44 +++++-- dbms/src/Storages/StorageMerge.cpp | 42 ++++++- dbms/src/Storages/StorageMergeTree.cpp | 41 ++++--- .../Storages/StorageReplicatedMergeTree.cpp | 25 ++-- dbms/src/Storages/StorageTinyLog.cpp | 32 +++-- dbms/src/Storages/StorageView.cpp | 28 ++++- 42 files changed, 797 insertions(+), 264 deletions(-) diff --git a/dbms/include/DB/Core/Block.h b/dbms/include/DB/Core/Block.h index 5446446dd1c..cc6c0094bf0 100644 --- a/dbms/include/DB/Core/Block.h +++ b/dbms/include/DB/Core/Block.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "ColumnsWithNameAndType.h" @@ -20,6 +21,8 @@ namespace DB * Позволяет вставлять, удалять столбцы в любом порядке, менять порядок столбцов. */ +class Context; + class Block { public: @@ -54,6 +57,7 @@ public: void insert(size_t position, const ColumnWithNameAndType & elem); /// вставить столбец в конец void insert(const ColumnWithNameAndType & elem); + void insertDefault(const String & name, const DataTypePtr & type); /// вставить столбец в конец, если столбца с таким именем ещё нет void insertUnique(const ColumnWithNameAndType & elem); /// удалить столбец в заданной позиции @@ -62,6 +66,8 @@ public: void erase(const String & name); /// Добавляет в блок недостающие столбцы со значениями по-умолчанию void addDefaults(NamesAndTypesListPtr required_columns); + void addDefaults(NamesAndTypesListPtr required_columns, + const ColumnDefaults & column_defaults, const Context & context); ColumnWithNameAndType & getByPosition(size_t position); const ColumnWithNameAndType & getByPosition(size_t position) const; diff --git a/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h b/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h index a41c3665684..eb9759bfa57 100644 --- a/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h +++ b/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h @@ -5,6 +5,7 @@ #include #include +#include namespace DB @@ -19,8 +20,11 @@ class AddingDefaultBlockInputStream : public IProfilingBlockInputStream public: AddingDefaultBlockInputStream( BlockInputStreamPtr input_, - NamesAndTypesListPtr required_columns_) - : required_columns(required_columns_) + NamesAndTypesListPtr required_columns_, + const ColumnDefaults & column_defaults_, + const Context & context_) + : required_columns(required_columns_), + column_defaults(column_defaults_), context(context_) { children.push_back(input_); } @@ -45,12 +49,14 @@ protected: Block res = children.back()->read(); if (!res) return res; - res.addDefaults(required_columns); + res.addDefaults(required_columns, column_defaults, context); return res; } private: NamesAndTypesListPtr required_columns; + const ColumnDefaults & column_defaults; + Context context; }; } diff --git a/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h b/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h index 5b37c205244..8089549d919 100644 --- a/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h +++ b/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h @@ -5,6 +5,8 @@ #include #include +#include +#include namespace DB @@ -19,15 +21,18 @@ class AddingDefaultBlockOutputStream : public IBlockOutputStream public: AddingDefaultBlockOutputStream( BlockOutputStreamPtr output_, - NamesAndTypesListPtr required_columns_) - : output(output_), required_columns(required_columns_) + NamesAndTypesListPtr required_columns_, + const ColumnDefaults & column_defaults_, + const Context & context_) + : output(output_), required_columns(required_columns_), + column_defaults(column_defaults_), context(context_) { } void write(const Block & block) override { Block res = block; - res.addDefaults(required_columns); + res.addDefaults(required_columns, column_defaults, context); output->write(res); } @@ -39,6 +44,8 @@ public: private: BlockOutputStreamPtr output; NamesAndTypesListPtr required_columns; + const ColumnDefaults & column_defaults; + Context context; }; diff --git a/dbms/include/DB/Functions/FunctionsString.h b/dbms/include/DB/Functions/FunctionsString.h index 9b036755d83..50a114b310c 100644 --- a/dbms/include/DB/Functions/FunctionsString.h +++ b/dbms/include/DB/Functions/FunctionsString.h @@ -12,6 +12,7 @@ #include #include #include +#include namespace DB @@ -973,13 +974,16 @@ public: + toString(arguments.size()) + ", should be 2.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - if (!typeid_cast(&*arguments[0]) && !typeid_cast(&*arguments[0])) - throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (!typeid_cast(&*arguments[1]) && !typeid_cast(&*arguments[1])) - throw Exception("Illegal type " + arguments[1]->getName() + " of second argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + for (const auto arg_idx : ext::range(0, arguments.size())) + { + const auto arg = arguments[arg_idx].get(); + if (!typeid_cast(arg) && + !typeid_cast(arg)) + throw Exception{ + "Illegal type " + arg->getName() + " of argument " + std::to_string(arg_idx) + " of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT + }; + } return new DataTypeString; } diff --git a/dbms/include/DB/Interpreters/InterpreterCreateQuery.h b/dbms/include/DB/Interpreters/InterpreterCreateQuery.h index 382efa912fa..7a1a5eb72e7 100644 --- a/dbms/include/DB/Interpreters/InterpreterCreateQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterCreateQuery.h @@ -25,13 +25,18 @@ public: /// Список столбцов с типами в AST. static ASTPtr formatColumns(const NamesAndTypesList & columns); + static ASTPtr formatColumns( + NamesAndTypesList columns, + const NamesAndTypesList & alias_columns, + const ColumnDefaults & column_defaults); private: /// AST в список столбцов с типами. Столбцы типа Nested развернуты в список настоящих столбцов. using ColumnsAndDefaults = std::pair; - ColumnsAndDefaults parseColumns(ASTPtr expression_list, const DataTypeFactory & data_type_factory); + ColumnsAndDefaults parseColumns(ASTPtr expression_list); - DataTypePtr deduceType(const ASTPtr & expr, const NamesAndTypesList & columns) const; + /// removes alias columns from the columns list and return them in a separate list + static NamesAndTypesList removeAliasColumns(ColumnsAndDefaults & columns_and_defaults); ASTPtr query_ptr; Context context; diff --git a/dbms/include/DB/Parsers/ASTWithAlias.h b/dbms/include/DB/Parsers/ASTWithAlias.h index 59f5acc746b..bbba7f984a4 100644 --- a/dbms/include/DB/Parsers/ASTWithAlias.h +++ b/dbms/include/DB/Parsers/ASTWithAlias.h @@ -21,4 +21,12 @@ public: void setAlias(const String & to) override { alias = to; } }; +/// helper for setting aliases and chaining result to other functions +inline ASTPtr setAlias(ASTPtr ast, const String & alias) { + dynamic_cast(*ast).alias = alias; + + return ast; +}; + + } diff --git a/dbms/include/DB/Parsers/ParserCreateQuery.h b/dbms/include/DB/Parsers/ParserCreateQuery.h index 97ff2c60ef8..ce6a51ae145 100644 --- a/dbms/include/DB/Parsers/ParserCreateQuery.h +++ b/dbms/include/DB/Parsers/ParserCreateQuery.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -114,7 +115,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, Pos end, ASTPtr ParserString s_default{"DEFAULT"}; ParserString s_materialized{"MATERIALIZED"}; ParserString s_alias{"ALIAS"}; - ParserExpressionElement expr_parser; + ParserTernaryOperatorExpression expr_parser; const auto begin = pos; const auto reset_pos_and_return = [&pos, begin] { diff --git a/dbms/include/DB/Storages/IStorage.h b/dbms/include/DB/Storages/IStorage.h index 70ed1b158b4..f086ae8abd7 100644 --- a/dbms/include/DB/Storages/IStorage.h +++ b/dbms/include/DB/Storages/IStorage.h @@ -62,7 +62,7 @@ public: /** Возвращает true, если хранилище поддерживает запросы с секцией FINAL. */ virtual bool supportsFinal() const { return false; } - + /** Возвращает true, если хранилище поддерживает запросы с секцией PREWHERE. */ virtual bool supportsPrewhere() const { return false; } @@ -245,7 +245,7 @@ public: virtual void shutdown() {} /** Возвращает владеющий указатель на себя. - */ + */ std::shared_ptr thisPtr() { std::shared_ptr res = this_ptr.lock(); @@ -257,7 +257,7 @@ public: return res; } - bool is_dropped; + bool is_dropped{false}; /// Поддерживается ли индекс в секции IN virtual bool supportsIndexForIn() const { return false; }; @@ -266,7 +266,7 @@ public: virtual bool checkData() const { throw DB::Exception("Check query is not supported for " + getName() + " storage"); } protected: - IStorage() : is_dropped(false) {} + using ITableDeclaration::ITableDeclaration; private: std::weak_ptr this_ptr; diff --git a/dbms/include/DB/Storages/ITableDeclaration.h b/dbms/include/DB/Storages/ITableDeclaration.h index d03d8ee4f33..5d7f3dafe26 100644 --- a/dbms/include/DB/Storages/ITableDeclaration.h +++ b/dbms/include/DB/Storages/ITableDeclaration.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -36,6 +37,9 @@ public: */ virtual bool hasRealColumn(const String & column_name) const; + NameAndTypePair getAliasColumn(const String & column_name) const; + bool hasAliasColumn(const String & column_name) const; + /** Получить описание любого столбца по его имени. */ virtual NameAndTypePair getColumn(const String & column_name) const; @@ -69,7 +73,15 @@ public: */ void check(const Block & block, bool need_all = false) const; - virtual ~ITableDeclaration() {} + virtual ~ITableDeclaration() = default; + + ITableDeclaration() = default; + ITableDeclaration(const NamesAndTypesList & alias_columns, const ColumnDefaults & column_defaults) + : alias_columns{alias_columns}, column_defaults{column_defaults} + {} + + NamesAndTypesList alias_columns{}; + ColumnDefaults column_defaults{}; }; } diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h index 7df4bfe391b..8a068b89764 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h @@ -580,6 +580,8 @@ public: * require_part_metadata - обязательно ли в директории с куском должны быть checksums.txt и columns.txt */ MergeTreeData( const String & full_path_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, const Context & context_, ASTPtr & primary_expr_ast_, const String & date_column_name_, @@ -613,14 +615,14 @@ public: { if (column_name == "_part") return NameAndTypePair("_part", new DataTypeString); if (column_name == "_part_index") return NameAndTypePair("_part_index", new DataTypeUInt64); - return getRealColumn(column_name); + return ITableDeclaration::getColumn(column_name); } bool hasColumn(const String &column_name) const { if (column_name == "_part") return true; if (column_name == "_part_index") return true; - return hasRealColumn(column_name); + return ITableDeclaration::hasColumn(column_name); } String getFullPath() const { return full_path; } diff --git a/dbms/include/DB/Storages/StorageChunkMerger.h b/dbms/include/DB/Storages/StorageChunkMerger.h index 33f63c5c059..409c42bdad5 100644 --- a/dbms/include/DB/Storages/StorageChunkMerger.h +++ b/dbms/include/DB/Storages/StorageChunkMerger.h @@ -8,7 +8,7 @@ namespace DB { - + /** То и дело объединяет таблицы, подходящие под регэксп, в таблицы типа Chunks. * После объндинения заменяет исходные таблицы таблицами типа ChunkRef. * При чтении ведет себя как таблица типа Merge. @@ -21,18 +21,20 @@ public: const std::string & this_database_,/// Имя БД для этой таблицы. const std::string & name_, /// Имя таблицы. NamesAndTypesListPtr columns_, /// Список столбцов. + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, const String & source_database_, /// В какой БД искать таблицы-источники. const String & table_name_regexp_, /// Регексп имён таблиц-источников. const std::string & destination_name_prefix_, /// Префикс имен создаваемых таблиц типа Chunks. size_t chunks_to_merge_, /// Сколько чанков сливать в одну группу. Context & context_); /// Известные таблицы. - + std::string getName() const { return "ChunkMerger"; } std::string getTableName() const { return name; } - + const NamesAndTypesList & getColumnsList() const { return *columns; } - NameAndTypePair getColumn(const String &column_name) const; - bool hasColumn(const String &column_name) const; + NameAndTypePair getColumn(const String & column_name) const; + bool hasColumn(const String & column_name) const; BlockInputStreams read( const Names & column_names, @@ -45,9 +47,9 @@ public: void shutdown(); Block getBlockWithVirtualColumns(const Storages & selected_tables) const; - + ~StorageChunkMerger(); - + private: String this_database; String name; @@ -58,13 +60,13 @@ private: size_t chunks_to_merge; Context & context; Settings settings; - + boost::thread merge_thread; Poco::Event cancel_merge_thread; - + Logger * log; volatile bool shutdown_called; - + /// Название виртуального столбца, отвечающего за имя таблицы, из которой идет чтение. (Например "_table") String _table_column_name; @@ -72,12 +74,14 @@ private: const std::string & this_database_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, const String & source_database_, const String & table_name_regexp_, const std::string & destination_name_prefix_, size_t chunks_to_merge_, Context & context_); - + void mergeThread(); bool maybeMergeSomething(); Storages selectChunksToMerge(); @@ -88,5 +92,5 @@ private: /// Нужно смотреть, залочив mutex из контекста. static TableNames currently_written_groups; }; - + } diff --git a/dbms/include/DB/Storages/StorageChunkRef.h b/dbms/include/DB/Storages/StorageChunkRef.h index 4b54ea61ecb..557c5edaeed 100644 --- a/dbms/include/DB/Storages/StorageChunkRef.h +++ b/dbms/include/DB/Storages/StorageChunkRef.h @@ -5,7 +5,7 @@ namespace DB { - + /** Ссылка на кусок данных в таблице типа Chunks. * Запись не поддерживается. */ @@ -13,10 +13,10 @@ class StorageChunkRef : public IStorage { public: static StoragePtr create(const std::string & name_, const Context & context_, const std::string & source_database_name_, const std::string & source_table_name_, bool attach); - + std::string getName() const { return "ChunkRef"; } std::string getTableName() const { return name; } - + const NamesAndTypesList & getColumnsList() const { return getSource().getColumnsList(); } /// В таблице, на которую мы ссылаемся, могут быть виртуальные столбцы. NameAndTypePair getColumn(const String &column_name) const { return getSource().getColumn(column_name); }; @@ -31,23 +31,23 @@ public: unsigned threads = 1); ASTPtr getCustomCreateQuery(const Context & context) const; - + void drop() override; - + String source_database_name; String source_table_name; bool checkData() const override; - + private: String name; const Context & context; - + StorageChunkRef(const std::string & name_, const Context & context_, const std::string & source_database_name_, const std::string & source_table_name_, bool attach); /// TODO: может быть, можно просто хранить указатель на родительскую таблицу? StorageChunks & getSource(); const StorageChunks & getSource() const; }; - + } diff --git a/dbms/include/DB/Storages/StorageChunks.h b/dbms/include/DB/Storages/StorageChunks.h index b3382fe9779..d2cf40a5070 100644 --- a/dbms/include/DB/Storages/StorageChunks.h +++ b/dbms/include/DB/Storages/StorageChunks.h @@ -7,7 +7,7 @@ namespace DB { - + /** Хранит несколько кусков данных. Читает из всех кусков. * Запись не поддерживается. Для записи используются таблицы типа ChunkMerger. * Таблицы типа ChunkRef могут ссылаться на отдельные куски внутри таблицы типа Chunks. @@ -20,17 +20,19 @@ class StorageChunks : public StorageLog using StorageLog::read; public: static StoragePtr create(const std::string & path_, - const std::string & name_, - const std::string & database_name_, - NamesAndTypesListPtr columns_, - Context & context_, - bool attach); - + const std::string & name_, + const std::string & database_name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + Context & context_, + bool attach); + void addReference(); void removeReference(); - + std::string getName() const { return "Chunks"; } - + BlockInputStreams read( const Names & column_names, ASTPtr query, @@ -53,14 +55,14 @@ public: BlockOutputStreamPtr writeToNewChunk( const std::string & chunk_name); - + /// Если бы запись была разрешена, непонятно, как назвать новый чанк. BlockOutputStreamPtr write( ASTPtr query) { throw Exception("Table doesn't support writing", ErrorCodes::NOT_IMPLEMENTED); } - + /// Переименование испортило бы целостность количества ссылок из таблиц ChunkRef. void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) { @@ -82,27 +84,29 @@ private: typedef std::vector ChunkNumToChunkName; String database_name; - + ChunkNumToMark chunk_num_to_marks; ChunkIndices chunk_indices; ChunkNumToChunkName chunk_names; - + CounterInFile reference_counter; Context & context; - + Logger * log; - + StorageChunks(const std::string & path_, - const std::string & name_, - const std::string & database_name_, - NamesAndTypesListPtr columns_, - Context & context_, - bool attach); - + const std::string & name_, + const std::string & database_name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + Context & context_, + bool attach); + void dropThis(); - + void loadIndex(); void appendChunkToIndex(const std::string & chunk_name, size_t mark); }; - + } diff --git a/dbms/include/DB/Storages/StorageDistributed.h b/dbms/include/DB/Storages/StorageDistributed.h index f1a543bcddd..da4c82606be 100644 --- a/dbms/include/DB/Storages/StorageDistributed.h +++ b/dbms/include/DB/Storages/StorageDistributed.h @@ -25,6 +25,8 @@ public: static StoragePtr create( const std::string & name_, /// Имя таблицы. NamesAndTypesListPtr columns_, /// Список столбцов. + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, const String & remote_database_, /// БД на удалённых серверах. const String & remote_table_, /// Имя таблицы на удалённых серверах. const String & cluster_name, @@ -88,6 +90,18 @@ private: const ASTPtr & sharding_key_ = nullptr, const String & data_path_ = String{}); + StorageDistributed( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + const String & remote_database_, + const String & remote_table_, + Cluster & cluster_, + Context & context_, + const ASTPtr & sharding_key_ = nullptr, + const String & data_path_ = String{}); + /// create directory monitor thread by subdirectory name void createDirectoryMonitor(const std::string & name); diff --git a/dbms/include/DB/Storages/StorageFactory.h b/dbms/include/DB/Storages/StorageFactory.h index f361e92dadc..8510159a999 100644 --- a/dbms/include/DB/Storages/StorageFactory.h +++ b/dbms/include/DB/Storages/StorageFactory.h @@ -23,6 +23,8 @@ public: Context & context, ASTPtr & query, NamesAndTypesListPtr columns, + const NamesAndTypesList & alias_columns, + const ColumnDefaults & column_defaults, bool attach) const; }; diff --git a/dbms/include/DB/Storages/StorageLog.h b/dbms/include/DB/Storages/StorageLog.h index 2cae126ca02..7a488e573b1 100644 --- a/dbms/include/DB/Storages/StorageLog.h +++ b/dbms/include/DB/Storages/StorageLog.h @@ -64,11 +64,11 @@ private: if (offset) plain.seek(offset); } - + ReadBufferFromFile plain; CompressedReadBuffer compressed; }; - + typedef std::map > FileStreams; FileStreams streams; @@ -97,7 +97,7 @@ private: { plain_offset = Poco::File(data_path).getSize(); } - + WriteBufferFromFile plain; CompressedWriteBuffer compressed; @@ -111,12 +111,12 @@ private: }; typedef std::vector > MarksForColumns; - + typedef std::map > FileStreams; FileStreams streams; - + typedef std::set OffsetColumns; - + WriteBufferFromFile marks_stream; /// Объявлен ниже lock, чтобы файл открывался при захваченном rwlock. void addStream(const String & name, const IDataType & type, size_t level = 0); @@ -139,8 +139,14 @@ public: * (корректность имён и путей не проверяется) * состоящую из указанных столбцов; создать файлы, если их нет. */ - static StoragePtr create(const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, size_t max_compress_block_size_ = DEFAULT_MAX_COMPRESS_BLOCK_SIZE); - + static StoragePtr create( + const std::string & path_, + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + size_t max_compress_block_size_ = DEFAULT_MAX_COMPRESS_BLOCK_SIZE); + std::string getName() const { return "Log"; } std::string getTableName() const { return name; } @@ -193,16 +199,22 @@ protected: throw Exception("There is no column " + _table_column_name + " in table " + getTableName(), ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); } - StorageLog(const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, size_t max_compress_block_size_); - + StorageLog( + const std::string & path_, + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + size_t max_compress_block_size_); + /// Прочитать файлы с засечками, если они ещё не прочитаны. /// Делается лениво, чтобы при большом количестве таблиц, сервер быстро стартовал. /// Нельзя вызывать с залоченным на запись rwlock. void loadMarks(); - + /// Можно вызывать при любом состоянии rwlock. size_t marksCount(); - + BlockInputStreams read( size_t from_mark, size_t to_mark, @@ -212,14 +224,14 @@ protected: QueryProcessingStage::Enum & processed_stage, size_t max_block_size = DEFAULT_BLOCK_SIZE, unsigned threads = 1); - + private: Files_t files; /// name -> data Names column_names; /// column_index -> name - + Poco::File marks_file; - + /// Порядок добавления файлов не должен меняться: он соответствует порядку столбцов в файле с засечками. void addFile(const String & column_name, const IDataType & type, size_t level = 0); diff --git a/dbms/include/DB/Storages/StorageMaterializedView.h b/dbms/include/DB/Storages/StorageMaterializedView.h index 4cea9d804e0..0add286fb49 100644 --- a/dbms/include/DB/Storages/StorageMaterializedView.h +++ b/dbms/include/DB/Storages/StorageMaterializedView.h @@ -9,8 +9,15 @@ namespace DB class StorageMaterializedView : public StorageView { public: - static StoragePtr create(const String & table_name_, const String & database_name_, - Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, bool attach_); + static StoragePtr create( + const String & table_name_, + const String & database_name_, + Context & context_, + ASTPtr & query_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + bool attach_); std::string getName() const { return "MaterializedView"; } std::string getInnerTableName() const { return ".inner." + table_name; } @@ -33,9 +40,15 @@ public: private: StoragePtr data; - StorageMaterializedView(const String & table_name_, const String & database_name_, - Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, bool attach_); - + StorageMaterializedView( + const String & table_name_, + const String & database_name_, + Context & context_, + ASTPtr & query_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + bool attach_); }; } diff --git a/dbms/include/DB/Storages/StorageMemory.h b/dbms/include/DB/Storages/StorageMemory.h index 00735cb7400..bd7027c05fc 100644 --- a/dbms/include/DB/Storages/StorageMemory.h +++ b/dbms/include/DB/Storages/StorageMemory.h @@ -30,7 +30,7 @@ public: res << ")"; return res.str(); } - + protected: Block readImpl(); private: @@ -62,7 +62,15 @@ friend class MemoryBlockInputStream; friend class MemoryBlockOutputStream; public: - static StoragePtr create(const std::string & name_, NamesAndTypesListPtr columns_); + static StoragePtr create( + const std::string & name_, + NamesAndTypesListPtr columns_); + + static StoragePtr create( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_); std::string getName() const { return "Memory"; } std::string getTableName() const { return name; } @@ -93,8 +101,17 @@ private: BlocksList data; Poco::FastMutex mutex; - - StorageMemory(const std::string & name_, NamesAndTypesListPtr columns_); + + StorageMemory( + const std::string & name_, + NamesAndTypesListPtr columns_); + + StorageMemory( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_); + }; } diff --git a/dbms/include/DB/Storages/StorageMerge.h b/dbms/include/DB/Storages/StorageMerge.h index 603555fe9a0..a8d5876f05f 100644 --- a/dbms/include/DB/Storages/StorageMerge.h +++ b/dbms/include/DB/Storages/StorageMerge.h @@ -25,6 +25,15 @@ public: const String & table_name_regexp_, /// Регексп имён таблиц-источников. const Context & context_); /// Известные таблицы. + static StoragePtr create( + const std::string & name_, /// Имя таблицы. + NamesAndTypesListPtr columns_, /// Список столбцов. + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + const String & source_database_, /// В какой БД искать таблицы-источники. + const String & table_name_regexp_, /// Регексп имён таблиц-источников. + const Context & context_); /// Известные таблицы. + std::string getName() const { return "Merge"; } std::string getTableName() const { return name; } bool supportsSampling() const { return true; } @@ -46,7 +55,7 @@ public: void drop() override {} void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) { name = new_table_name; } - + /// в подтаблицах добавлять и удалять столбы нужно вручную /// структура подтаблиц не проверяется void alter(const AlterCommands & params, const String & database_name, const String & table_name, Context & context); @@ -58,7 +67,7 @@ private: String source_database; OptimizedRegularExpression table_name_regexp; const Context & context; - + StorageMerge( const std::string & name_, NamesAndTypesListPtr columns_, @@ -66,6 +75,15 @@ private: const String & table_name_regexp_, const Context & context_); + StorageMerge( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + const String & source_database_, + const String & table_name_regexp_, + const Context & context_); + void getSelectedTables(StorageVector & selected_tables) const; }; diff --git a/dbms/include/DB/Storages/StorageMergeTree.h b/dbms/include/DB/Storages/StorageMergeTree.h index 725076fe9e1..4c3dd9a07e8 100644 --- a/dbms/include/DB/Storages/StorageMergeTree.h +++ b/dbms/include/DB/Storages/StorageMergeTree.h @@ -24,8 +24,13 @@ public: * date_column_name - имя столбца с датой; * index_granularity - на сколько строчек пишется одно значение индекса. */ - static StoragePtr create(const String & path_, const String & database_name_, const String & table_name_, + static StoragePtr create( + const String & path_, + const String & database_name_, + const String & table_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, Context & context_, ASTPtr & primary_expr_ast_, const String & date_column_name_, @@ -50,12 +55,12 @@ public: const NamesAndTypesList & getColumnsList() const { return data.getColumnsList(); } - NameAndTypePair getColumn(const String &column_name) const + NameAndTypePair getColumn(const String & column_name) const { return data.getColumn(column_name); } - bool hasColumn(const String &column_name) const + bool hasColumn(const String & column_name) const { return data.hasColumn(column_name); } @@ -153,16 +158,21 @@ private: typedef Poco::SharedPtr CurrentlyMergingPartsTaggerPtr; - StorageMergeTree(const String & path_, const String & database_name_, const String & table_name_, - NamesAndTypesListPtr columns_, - Context & context_, - ASTPtr & primary_expr_ast_, - const String & date_column_name_, - const ASTPtr & sampling_expression_, /// nullptr, если семплирование не поддерживается. - size_t index_granularity_, - MergeTreeData::Mode mode_, - const String & sign_column_, - const MergeTreeSettings & settings_); + StorageMergeTree( + const String & path_, + const String & database_name_, + const String & table_name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + Context & context_, + ASTPtr & primary_expr_ast_, + const String & date_column_name_, + const ASTPtr & sampling_expression_, /// nullptr, если семплирование не поддерживается. + size_t index_granularity_, + MergeTreeData::Mode mode_, + const String & sign_column_, + const MergeTreeSettings & settings_); /** Определяет, какие куски нужно объединять, и объединяет их. * Если aggressive - выбрать куски, не обращая внимание на соотношение размеров и их новизну (для запроса OPTIMIZE). diff --git a/dbms/include/DB/Storages/StorageNull.h b/dbms/include/DB/Storages/StorageNull.h index 62fd8778c9f..4d2c79f0597 100644 --- a/dbms/include/DB/Storages/StorageNull.h +++ b/dbms/include/DB/Storages/StorageNull.h @@ -15,9 +15,13 @@ namespace DB class StorageNull : public IStorage { public: - static StoragePtr create(const std::string & name_, NamesAndTypesListPtr columns_) + static StoragePtr create( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_) { - return (new StorageNull(name_, columns_))->thisPtr(); + return (new StorageNull{name_, columns_, alias_columns_, column_defaults_})->thisPtr(); } std::string getName() const { return "Null"; } @@ -48,8 +52,12 @@ private: String name; NamesAndTypesListPtr columns; - StorageNull(const std::string & name_, NamesAndTypesListPtr columns_) - : name(name_), columns(columns_) {} + StorageNull( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_) + : IStorage{alias_columns_, column_defaults_}, name(name_), columns(columns_) {} }; } diff --git a/dbms/include/DB/Storages/StorageReplicatedMergeTree.h b/dbms/include/DB/Storages/StorageReplicatedMergeTree.h index b07debb1486..067f9a04aa5 100644 --- a/dbms/include/DB/Storages/StorageReplicatedMergeTree.h +++ b/dbms/include/DB/Storages/StorageReplicatedMergeTree.h @@ -28,6 +28,8 @@ public: bool attach, const String & path_, const String & database_name_, const String & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, Context & context_, ASTPtr & primary_expr_ast_, const String & date_column_name_, @@ -52,13 +54,13 @@ public: const NamesAndTypesList & getColumnsList() const override { return data.getColumnsList(); } - NameAndTypePair getColumn(const String &column_name) const + NameAndTypePair getColumn(const String & column_name) const { if (column_name == "_replicated") return NameAndTypePair("_replicated", new DataTypeUInt8); return data.getColumn(column_name); } - bool hasColumn(const String &column_name) const + bool hasColumn(const String & column_name) const { if (column_name == "_replicated") return true; return data.hasColumn(column_name); @@ -319,6 +321,8 @@ private: bool attach, const String & path_, const String & database_name_, const String & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, Context & context_, ASTPtr & primary_expr_ast_, const String & date_column_name_, diff --git a/dbms/include/DB/Storages/StorageTinyLog.h b/dbms/include/DB/Storages/StorageTinyLog.h index 770259f69d4..9e57f7b780b 100644 --- a/dbms/include/DB/Storages/StorageTinyLog.h +++ b/dbms/include/DB/Storages/StorageTinyLog.h @@ -29,7 +29,7 @@ public: String getName() const { return "TinyLogBlockInputStream"; } String getID() const; - + protected: Block readImpl(); private: @@ -89,7 +89,7 @@ private: typedef std::map > FileStreams; FileStreams streams; - + typedef std::set OffsetColumns; void addStream(const String & name, const IDataType & type, size_t level = 0); @@ -111,7 +111,14 @@ public: * состоящую из указанных столбцов. * Если не указано attach - создать директорию, если её нет. */ - static StoragePtr create(const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, bool attach, size_t max_compress_block_size_ = DEFAULT_MAX_COMPRESS_BLOCK_SIZE); + static StoragePtr create( + const std::string & path_, + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + bool attach, + size_t max_compress_block_size_ = DEFAULT_MAX_COMPRESS_BLOCK_SIZE); std::string getName() const { return "TinyLog"; } std::string getTableName() const { return name; } @@ -130,7 +137,7 @@ public: ASTPtr query); void drop() override; - + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name); bool checkData() const override; @@ -144,7 +151,7 @@ public: Files_t & getFiles(); - std::string full_path() { return path + escapeForFileName(name) + '/';} + std::string full_path() { return path + escapeForFileName(name) + '/';} private: String path; @@ -159,8 +166,15 @@ private: Logger * log; - StorageTinyLog(const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, bool attach, size_t max_compress_block_size_); - + StorageTinyLog( + const std::string & path_, + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + bool attach, + size_t max_compress_block_size_); + void addFile(const String & column_name, const IDataType & type, size_t level = 0); }; diff --git a/dbms/include/DB/Storages/StorageView.h b/dbms/include/DB/Storages/StorageView.h index 547b38049f4..53b6767391f 100644 --- a/dbms/include/DB/Storages/StorageView.h +++ b/dbms/include/DB/Storages/StorageView.h @@ -10,8 +10,14 @@ namespace DB class StorageView : public IStorage { public: - static StoragePtr create(const String & table_name_, const String & database_name_, - Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_); + static StoragePtr create( + const String & table_name_, + const String & database_name_, + Context & context_, + ASTPtr & query_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_); virtual std::string getName() const { return "View"; } virtual std::string getTableName() const { return table_name; } @@ -37,8 +43,14 @@ protected: Context & context; NamesAndTypesListPtr columns; - StorageView(const String & table_name_, const String & database_name_, - Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_); + StorageView( + const String & table_name_, + const String & database_name_, + Context & context_, + ASTPtr & query_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_); }; } diff --git a/dbms/include/DB/TableFunctions/TableFunctionMerge.h b/dbms/include/DB/TableFunctions/TableFunctionMerge.h index 467604577f1..719b6470110 100644 --- a/dbms/include/DB/TableFunctions/TableFunctionMerge.h +++ b/dbms/include/DB/TableFunctions/TableFunctionMerge.h @@ -48,7 +48,8 @@ public: /// Нам необходимо его пометить как имя базы данных, поскольку по умолчанию стоит значение column typeid_cast(*args[0]).kind = ASTIdentifier::Database; - return StorageMerge::create(getName(), chooseColumns(source_database, table_name_regexp, context), source_database, table_name_regexp, context); + return StorageMerge::create(getName(), chooseColumns(source_database, table_name_regexp, context), + source_database, table_name_regexp, context); } private: diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index 878c67d838e..327766b4bb6 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -8,6 +8,12 @@ #include #include +#include +#include +#include + +#include + namespace DB { @@ -21,18 +27,42 @@ Block::Block(const Block & other) void Block::addDefaults(NamesAndTypesListPtr required_columns) { - for (NamesAndTypesList::const_iterator it = required_columns->begin(); it != required_columns->end(); ++it) + for (const auto & column : *required_columns) + if (!has(column.name)) + insertDefault(column.name, column.type); +} + +void Block::addDefaults(NamesAndTypesListPtr required_columns, + const ColumnDefaults & column_defaults, const Context & context) +{ + ASTPtr default_expr_list{stdext::make_unique().release()}; + + for (const auto & column : *required_columns) { - if (!has(it->name)) - { - ColumnWithNameAndType col; - col.name = it->name; - col.type = it->type; - col.column = dynamic_cast(*it->type->createConstColumn( - rows(), it->type->getDefault())).convertToFullColumn(); - insert(col); - } + if (has(column.name)) + continue; + + const auto it = column_defaults.find(column.name); + + if (it == column_defaults.end()) + insertDefault(column.name, column.type); + else + default_expr_list->children.emplace_back(it->second.expression); } + + /// nothing to evaluate + if (default_expr_list->children.empty()) + return; + + /** ExpressionAnalyzer eliminates "unused" columns, in order to ensure theri safety + * we are going to operate on a copy instead of original block */ + Block copy_block{*this}; + /// evaluate default values for defaulted columns + ExpressionAnalyzer{default_expr_list, context, *required_columns}.getActions(true)->execute(copy_block); + + /// move evaluated columns to original block + for (auto & column_name_type : copy_block.getColumns()) + insert(std::move(column_name_type)); } Block & Block::operator= (const Block & other) @@ -82,6 +112,16 @@ void Block::insert(const ColumnWithNameAndType & elem) index_by_position.push_back(it); } +void Block::insertDefault(const String & name, const DataTypePtr & type) +{ + insert({ + dynamic_cast(*type->createConstColumn(rows(), + type->getDefault())).convertToFullColumn(), + type, name + }); +} + + void Block::insertUnique(const ColumnWithNameAndType & elem) { diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index f9518dfa0e1..46b03c7cba1 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -89,6 +89,7 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) StoragePtr res; String storage_name; NamesAndTypesListPtr columns = new NamesAndTypesList; + NamesAndTypesList alias_columns{}; ColumnDefaults column_defaults{}; StoragePtr as_storage; @@ -118,7 +119,8 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) /// Получаем список столбцов if (create.columns) { - auto && columns_and_defaults = parseColumns(create.columns, context.getDataTypeFactory()); + auto && columns_and_defaults = parseColumns(create.columns); + alias_columns = removeAliasColumns(columns_and_defaults); columns = new NamesAndTypesList{std::move(columns_and_defaults.first)}; column_defaults = std::move(columns_and_defaults.second); } @@ -134,7 +136,7 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY); /// Даже если в запросе был список столбцов, на всякий случай приведем его к стандартному виду (развернем Nested). - ASTPtr new_columns = formatColumns(*columns); + ASTPtr new_columns = formatColumns(*columns, alias_columns, column_defaults); if (create.columns) { auto it = std::find(create.children.begin(), create.children.end(), create.columns); @@ -183,7 +185,8 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) res = context.getStorageFactory().get( storage_name, data_path, table_name, database_name, context, - context.getGlobalContext(), query_ptr, columns, create.attach); + context.getGlobalContext(), query_ptr, + columns, alias_columns, column_defaults, create.attach); /// Проверка наличия метаданных таблицы на диске и создание метаданных if (!assume_metadata_exists && !create.is_temporary) @@ -238,32 +241,23 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) return res; } -InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( - ASTPtr expression_list, const DataTypeFactory & data_type_factory) +InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns(ASTPtr expression_list) { + auto & column_list_ast = typeid_cast(*expression_list); + /// list of table columns in correct order - NamesAndTypesList columns{}; + NamesAndTypesList columns{}, known_type_columns{}; ColumnDefaults defaults{}; - auto & columns_list = typeid_cast(*expression_list); - - using postprocess_column = std::pair; /// Columns requiring type-deduction or default_expression type-check - std::vector defaulted_columns{}; + std::vector> defaulted_columns{}; /** all default_expressions as a single expression list, * mixed with conversion-columns for each explicitly specified type */ ASTPtr default_expr_list{new ASTExpressionList}; - default_expr_list->children.reserve(columns_list.children.size()); + default_expr_list->children.reserve(column_list_ast.children.size()); - /// helper for setting aliases and chaining result to other functions - const auto set_alias = [] (ASTPtr ast, const String & alias) { - dynamic_cast(*ast).alias = alias; - - return ast; - }; - - for (auto & ast : columns_list.children) + for (auto & ast : column_list_ast.children) { auto & col_decl = typeid_cast(*ast); @@ -271,7 +265,8 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( { const auto & type_range = col_decl.type->range; columns.emplace_back(col_decl.name, - data_type_factory.get({ type_range.first, type_range.second })); + context.getDataTypeFactory().get({ type_range.first, type_range.second })); + known_type_columns.emplace_back(columns.back()); } else columns.emplace_back(col_decl.name, nullptr); @@ -290,18 +285,18 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( const auto & final_column_name = col_decl.name; const auto conversion_function_name = "to" + columns.back().type->getName(); - default_expr_list->children.emplace_back(set_alias( + default_expr_list->children.emplace_back(setAlias( makeASTFunction(conversion_function_name, ASTPtr{new ASTIdentifier{{}, tmp_column_name}}), final_column_name)); - default_expr_list->children.emplace_back(set_alias(col_decl.default_expression->clone(), tmp_column_name)); + default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), tmp_column_name)); } else { - default_expr_list->children.emplace_back(set_alias(col_decl.default_expression->clone(), col_decl.name)); + default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), col_decl.name)); defaults.emplace(col_decl.name, ColumnDefault{ columnDefaultTypeFromString(col_decl.default_specifier), - col_decl.default_expression + setAlias(col_decl.default_expression, col_decl.name) }); } } @@ -310,7 +305,7 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( /// set missing types and wrap default_expression's in a conversion-function if necessary if (!defaulted_columns.empty()) { - const auto actions = ExpressionAnalyzer{default_expr_list, context, columns}.getActions(true); + const auto actions = ExpressionAnalyzer{default_expr_list, context, known_type_columns}.getActions(true); const auto block = actions->getSampleBlock(); for (auto & column : defaulted_columns) @@ -339,7 +334,7 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( defaults.emplace(col_decl_ptr->name, ColumnDefault{ columnDefaultTypeFromString(col_decl_ptr->default_specifier), - col_decl_ptr->default_expression + setAlias(col_decl_ptr->default_expression, col_decl_ptr->name) }); } } @@ -347,6 +342,28 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( return { *DataTypeNested::expandNestedColumns(columns), defaults }; } +NamesAndTypesList InterpreterCreateQuery::removeAliasColumns(ColumnsAndDefaults & columns_and_defaults) +{ + auto & columns = columns_and_defaults.first; + auto & defaults = columns_and_defaults.second; + + NamesAndTypesList alias_columns{}; + + for (auto it = std::begin(columns); it != std::end(columns);) + { + const auto jt = defaults.find(it->name); + if (jt != std::end(defaults) && jt->second.type != ColumnDefaultType::Default) + { + alias_columns.push_back(*it); + it = columns.erase(it); + } + else + ++it; + } + + return alias_columns; +} + ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns) { ASTPtr columns_list_ptr{new ASTExpressionList}; @@ -375,11 +392,45 @@ ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns) return columns_list_ptr; } -DataTypePtr InterpreterCreateQuery::deduceType(const ASTPtr & expr, const NamesAndTypesList & columns) const +ASTPtr InterpreterCreateQuery::formatColumns(NamesAndTypesList columns, + const NamesAndTypesList & alias_columns, + const ColumnDefaults & column_defaults) { - const auto actions = ExpressionAnalyzer{expr, context, columns}.getActions(false); + columns.insert(std::end(columns), std::begin(alias_columns), std::end(alias_columns)); - return actions->getSampleBlock().getByName(expr->getColumnName()).type; + ASTPtr columns_list_ptr{new ASTExpressionList}; + ASTExpressionList & columns_list = typeid_cast(*columns_list_ptr); + + for (const auto & column : columns) + { + const auto column_declaration = new ASTColumnDeclaration; + ASTPtr column_declaration_ptr{column_declaration}; + + column_declaration->name = column.name; + + StringPtr type_name{new String(column.type->getName())}; + auto pos = type_name->data(); + const auto end = pos + type_name->size(); + + ParserIdentifierWithOptionalParameters storage_p; + Expected expected{""}; + if (!storage_p.parse(pos, end, column_declaration->type, expected)) + throw Exception("Cannot parse data type.", ErrorCodes::SYNTAX_ERROR); + + column_declaration->type->query_string = type_name; + + const auto it = column_defaults.find(column.name); + if (it != std::end(column_defaults)) + { + column_declaration->default_specifier = toString(it->second.type); + column_declaration->default_expression = setAlias(it->second.expression->clone(), ""); + } + + columns_list.children.push_back(column_declaration_ptr); + } + + return columns_list_ptr; } + } diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 59fc38c6a91..cefca2424aa 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -75,7 +75,10 @@ void InterpreterInsertQuery::execute(ReadBuffer * remaining_data_istr) table->write(query_ptr); /// Создаем кортеж из нескольких стримов, в которые будем писать данные. - BlockOutputStreamPtr out = new AddingDefaultBlockOutputStream(new PushingToViewsBlockOutputStream(query.database, query.table, context, query_ptr), required_columns); + BlockOutputStreamPtr out{new AddingDefaultBlockOutputStream{ + new PushingToViewsBlockOutputStream{query.database, query.table, context, query_ptr}, + required_columns, table->column_defaults, context + }}; /// Какой тип запроса: INSERT VALUES | INSERT FORMAT | INSERT SELECT? if (!query.select) @@ -128,7 +131,10 @@ BlockIO InterpreterInsertQuery::execute() table->write(query_ptr); /// Создаем кортеж из нескольких стримов, в которые будем писать данные. - BlockOutputStreamPtr out = new AddingDefaultBlockOutputStream(new PushingToViewsBlockOutputStream(query.database, query.table, context, query_ptr), required_columns); + BlockOutputStreamPtr out{new AddingDefaultBlockOutputStream{ + new PushingToViewsBlockOutputStream{query.database, query.table, context, query_ptr}, + required_columns, table->column_defaults, context + }}; BlockIO res; res.out_sample = getSampleBlock(); diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index 0468ba13451..c81fb242513 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -40,16 +40,37 @@ NameAndTypePair ITableDeclaration::getRealColumn(const String & column_name) con throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); } +NameAndTypePair ITableDeclaration::getAliasColumn(const String & column_name) const +{ + for (auto & column : alias_columns) + if (column.name == column_name) + return column; + + throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); +} + +bool ITableDeclaration::hasAliasColumn(const String & column_name) const +{ + for (auto & column : alias_columns) + if (column.name == column_name) + return true; + + return false; +} bool ITableDeclaration::hasColumn(const String & column_name) const { - return hasRealColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. + return hasRealColumn(column_name) || hasAliasColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. } - NameAndTypePair ITableDeclaration::getColumn(const String & column_name) const { - return getRealColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. + const auto it = column_defaults.find(column_name); + + if (it == std::end(column_defaults) || it->second.type == ColumnDefaultType::Default) + return getRealColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. + + return getAliasColumn(column_name); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index bc967d48cd3..4771ad00eed 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -21,6 +21,8 @@ namespace DB MergeTreeData::MergeTreeData( const String & full_path_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, const Context & context_, ASTPtr & primary_expr_ast_, const String & date_column_name_, const ASTPtr & sampling_expression_, @@ -31,7 +33,7 @@ MergeTreeData::MergeTreeData( const String & log_name_, bool require_part_metadata_, BrokenPartCallback broken_part_callback_) - : context(context_), + : ITableDeclaration{alias_columns_, column_defaults_}, context(context_), date_column_name(date_column_name_), sampling_expression(sampling_expression_), index_granularity(index_granularity_), mode(mode_), sign_column(sign_column_), diff --git a/dbms/src/Storages/StorageChunkMerger.cpp b/dbms/src/Storages/StorageChunkMerger.cpp index c9492fa806f..02014503be2 100644 --- a/dbms/src/Storages/StorageChunkMerger.cpp +++ b/dbms/src/Storages/StorageChunkMerger.cpp @@ -30,25 +30,31 @@ StoragePtr StorageChunkMerger::create( const std::string & this_database_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, const String & source_database_, const String & table_name_regexp_, const std::string & destination_name_prefix_, size_t chunks_to_merge_, Context & context_) { - return (new StorageChunkMerger(this_database_, name_, columns_, source_database_, table_name_regexp_, destination_name_prefix_, chunks_to_merge_, context_))->thisPtr(); + return (new StorageChunkMerger{ + this_database_, name_, columns_, alias_columns_, column_defaults_, + source_database_, table_name_regexp_, destination_name_prefix_, + chunks_to_merge_, context_ + })->thisPtr(); } -NameAndTypePair StorageChunkMerger::getColumn(const String &column_name) const +NameAndTypePair StorageChunkMerger::getColumn(const String & column_name) const { if (column_name == _table_column_name) return NameAndTypePair(_table_column_name, new DataTypeString); - return getRealColumn(column_name); + return IStorage::getColumn(column_name); } -bool StorageChunkMerger::hasColumn(const String &column_name) const +bool StorageChunkMerger::hasColumn(const String & column_name) const { if (column_name == _table_column_name) return true; - return hasRealColumn(column_name); + return IStorage::hasColumn(column_name); } BlockInputStreams StorageChunkMerger::read( @@ -221,12 +227,15 @@ StorageChunkMerger::StorageChunkMerger( const std::string & this_database_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, const String & source_database_, const String & table_name_regexp_, const std::string & destination_name_prefix_, size_t chunks_to_merge_, Context & context_) - : this_database(this_database_), name(name_), columns(columns_), source_database(source_database_), + : IStorage{alias_columns_, column_defaults_}, + this_database(this_database_), name(name_), columns(columns_), source_database(source_database_), table_name_regexp(table_name_regexp_), destination_name_prefix(destination_name_prefix_), chunks_to_merge(chunks_to_merge_), context(context_), settings(context.getSettings()), log(&Logger::get("StorageChunkMerger")), shutdown_called(false) @@ -457,7 +466,10 @@ bool StorageChunkMerger::mergeChunks(const Storages & chunks) processed_stage, DEFAULT_MERGE_BLOCK_SIZE); - BlockInputStreamPtr input = new AddingDefaultBlockInputStream(new ConcatBlockInputStream(input_streams), required_columns); + BlockInputStreamPtr input{new AddingDefaultBlockInputStream{ + new ConcatBlockInputStream{input_streams}, + required_columns, src_storage->column_defaults, context + }}; input->readPrefix(); output->writePrefix(); diff --git a/dbms/src/Storages/StorageChunks.cpp b/dbms/src/Storages/StorageChunks.cpp index f8bc1b8ad1e..05b9b6e376e 100644 --- a/dbms/src/Storages/StorageChunks.cpp +++ b/dbms/src/Storages/StorageChunks.cpp @@ -10,16 +10,22 @@ namespace DB { - + StoragePtr StorageChunks::create( const std::string & path_, const std::string & name_, const std::string & database_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, Context & context_, bool attach) { - return (new StorageChunks(path_, name_, database_name_, columns_, context_, attach))->thisPtr(); + return (new StorageChunks{ + path_, name_, database_name_, + columns_, alias_columns_, column_defaults_, + context_, attach + })->thisPtr(); } void StorageChunks::addReference() @@ -92,10 +98,10 @@ BlockInputStreams StorageChunks::readFromChunk( { size_t mark1; size_t mark2; - + { Poco::ScopedReadRWLock lock(rwlock); - + if (!chunk_indices.count(chunk_name)) throw Exception("No chunk " + chunk_name + " in table " + name, ErrorCodes::CHUNK_NOT_FOUND); size_t index = chunk_indices[chunk_name]; @@ -105,7 +111,7 @@ BlockInputStreams StorageChunks::readFromChunk( return read(mark1, mark2, column_names, query, settings, processed_stage, max_block_size, threads); } - + BlockOutputStreamPtr StorageChunks::writeToNewChunk( const std::string & chunk_name) { @@ -121,19 +127,21 @@ BlockOutputStreamPtr StorageChunks::writeToNewChunk( chunk_num_to_marks.push_back(mark); chunk_names.push_back(chunk_name); } - + return StorageLog::write(nullptr); } - + StorageChunks::StorageChunks( const std::string & path_, const std::string & name_, const std::string & database_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, Context & context_, bool attach) : - StorageLog(path_, name_, columns_, context_.getSettings().max_compress_block_size), + StorageLog(path_, name_, columns_, alias_columns_, column_defaults_, context_.getSettings().max_compress_block_size), database_name(database_name_), reference_counter(path_ + escapeForFileName(name_) + "/refcount.txt"), context(context_), @@ -184,15 +192,15 @@ std::pair StorageChunks::getTableFromMark(size_t mark) const last = chunk_num_to_marks[pos + 1] - 1; return std::make_pair(chunk_names[pos], last); } - + void StorageChunks::loadIndex() { loadMarks(); Poco::ScopedWriteRWLock lock(rwlock); - + String index_path = path + escapeForFileName(name) + "/chunks.chn"; - + if (!Poco::File(index_path).exists()) return; @@ -201,10 +209,10 @@ void StorageChunks::loadIndex() { String name; size_t mark; - + readStringBinary(name, index); readIntBinary(mark, index); - + chunk_indices[name] = chunk_num_to_marks.size(); chunk_num_to_marks.push_back(mark); chunk_names.push_back(name); @@ -224,14 +232,14 @@ void StorageChunks::appendChunkToIndex(const std::string & chunk_name, size_t ma void StorageChunks::dropThis() { LOG_TRACE(log, "Table " << name << " will drop itself."); - + ASTDropQuery * query = new ASTDropQuery(); ASTPtr query_ptr = query; query->detach = false; query->if_exists = false; query->database = database_name; query->table = name; - + InterpreterDropQuery interpreter(query_ptr, context); interpreter.execute(); } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 5d7074c7512..242bf3b335e 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -70,9 +70,34 @@ StorageDistributed::StorageDistributed( createDirectoryMonitors(); } +StorageDistributed::StorageDistributed( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + const String & remote_database_, + const String & remote_table_, + Cluster & cluster_, + Context & context_, + const ASTPtr & sharding_key_, + const String & data_path_) + : IStorage{alias_columns_, column_defaults_}, + name(name_), columns(columns_), + remote_database(remote_database_), remote_table(remote_table_), + context(context_), cluster(cluster_), + sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, *columns).getActions(false) : nullptr), + sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}), + write_enabled(cluster.getLocalNodesNum() + cluster.pools.size() < 2 || sharding_key_), + path(data_path_ + escapeForFileName(name) + '/') +{ + createDirectoryMonitors(); +} + StoragePtr StorageDistributed::create( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, const String & remote_database_, const String & remote_table_, const String & cluster_name, @@ -83,7 +108,8 @@ StoragePtr StorageDistributed::create( context_.initClusters(); return (new StorageDistributed{ - name_, columns_, remote_database_, remote_table_, + name_, columns_, alias_columns_, column_defaults_, + remote_database_, remote_table_, context_.getCluster(cluster_name), context_, sharding_key_, data_path_ })->thisPtr(); @@ -100,7 +126,8 @@ StoragePtr StorageDistributed::create( { auto res = new StorageDistributed{ name_, columns_, remote_database_, - remote_table_, *owned_cluster_, context_}; + remote_table_, *owned_cluster_, context_ + }; /// Захватываем владение объектом-кластером. res->owned_cluster = owned_cluster_; diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 49c5625b871..507e5853e50 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -72,15 +72,23 @@ StoragePtr StorageFactory::get( Context & context, ASTPtr & query, NamesAndTypesListPtr columns, + const NamesAndTypesList & alias_columns, + const ColumnDefaults & column_defaults, bool attach) const { if (name == "Log") { - return StorageLog::create(data_path, table_name, columns, context.getSettings().max_compress_block_size); + return StorageLog::create( + data_path, table_name, + columns, alias_columns, column_defaults, + context.getSettings().max_compress_block_size); } else if (name == "Chunks") { - return StorageChunks::create(data_path, table_name, database_name, columns, context, attach); + return StorageChunks::create( + data_path, table_name, database_name, + columns, alias_columns, column_defaults, + context, attach); } else if (name == "ChunkRef") { @@ -88,11 +96,16 @@ StoragePtr StorageFactory::get( } else if (name == "View") { - return StorageView::create(table_name, database_name, context, query, columns); + return StorageView::create( + table_name, database_name, context, query, + columns, alias_columns, column_defaults); } else if (name == "MaterializedView") { - return StorageMaterializedView::create(table_name, database_name, context, query, columns, attach); + return StorageMaterializedView::create( + table_name, database_name, context, query, + columns, alias_columns, column_defaults, + attach); } else if (name == "ChunkMerger") { @@ -118,8 +131,12 @@ StoragePtr StorageFactory::get( if (args.size() > 3) destination_name_prefix = typeid_cast(*args[3]).name; - return StorageChunkMerger::create(database_name, table_name, columns, source_database, source_table_name_regexp, destination_name_prefix, chunks_to_merge, context); - } while(false); + return StorageChunkMerger::create( + database_name, table_name, + columns, alias_columns, column_defaults, + source_database, source_table_name_regexp, + destination_name_prefix, chunks_to_merge, context); + } while (false); throw Exception("Storage ChunkMerger requires from 3 to 4 parameters:" " source database, regexp for source table names, number of chunks to merge, [destination tables name prefix].", @@ -127,15 +144,18 @@ StoragePtr StorageFactory::get( } else if (name == "TinyLog") { - return StorageTinyLog::create(data_path, table_name, columns, attach, context.getSettings().max_compress_block_size); + return StorageTinyLog::create( + data_path, table_name, + columns, alias_columns, column_defaults, + attach, context.getSettings().max_compress_block_size); } else if (name == "Memory") { - return StorageMemory::create(table_name, columns); + return StorageMemory::create(table_name, columns, alias_columns, column_defaults); } else if (name == "Null") { - return StorageNull::create(table_name, columns); + return StorageNull::create(table_name, columns, alias_columns, column_defaults); } else if (name == "Merge") { @@ -159,7 +179,9 @@ StoragePtr StorageFactory::get( String source_database = reinterpretAsIdentifier(args[0], local_context).name; String table_name_regexp = safeGet(typeid_cast(*args[1]).value); - return StorageMerge::create(table_name, columns, source_database, table_name_regexp, context); + return StorageMerge::create( + table_name, columns, alias_columns, column_defaults, + source_database, table_name_regexp, context); } else if (name == "Distributed") { @@ -188,7 +210,9 @@ StoragePtr StorageFactory::get( const auto & sharding_key = args.size() == 4 ? args[3] : nullptr; return StorageDistributed::create( - table_name, columns, remote_database, remote_table, cluster_name, context, sharding_key, data_path); + table_name, columns, alias_columns, column_defaults, + remote_database, remote_table, cluster_name, + context, sharding_key, data_path); } else if (endsWith(name, "MergeTree")) { @@ -302,12 +326,16 @@ StoragePtr StorageFactory::get( throw Exception("Index granularity must be a positive integer", ErrorCodes::BAD_ARGUMENTS); if (replicated) - return StorageReplicatedMergeTree::create(zookeeper_path, replica_name, attach, data_path, database_name, table_name, - columns, context, primary_expr_list, date_column_name, + return StorageReplicatedMergeTree::create( + zookeeper_path, replica_name, attach, data_path, database_name, table_name, + columns, alias_columns, column_defaults, + context, primary_expr_list, date_column_name, sampling_expression, index_granularity, mode, sign_column_name); else - return StorageMergeTree::create(data_path, database_name, table_name, - columns, context, primary_expr_list, date_column_name, + return StorageMergeTree::create( + data_path, database_name, table_name, + columns, alias_columns, column_defaults, + context, primary_expr_list, date_column_name, sampling_expression, index_granularity, mode, sign_column_name); } else diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 6a7cccc7de7..ccd26e17b20 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -409,8 +409,16 @@ void LogBlockOutputStream::writeMarks(MarksForColumns marks) } -StorageLog::StorageLog(const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, size_t max_compress_block_size_) - : path(path_), name(name_), columns(columns_), loaded_marks(false), max_compress_block_size(max_compress_block_size_), +StorageLog::StorageLog( + const std::string & path_, + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + size_t max_compress_block_size_) + : IStorage{alias_columns_, column_defaults_}, + path(path_), name(name_), columns(columns_), + loaded_marks(false), max_compress_block_size(max_compress_block_size_), file_checker(path + escapeForFileName(name) + '/' + "sizes.json", *this) { if (columns->empty()) @@ -425,9 +433,19 @@ StorageLog::StorageLog(const std::string & path_, const std::string & name_, Nam marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); } -StoragePtr StorageLog::create(const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, size_t max_compress_block_size_) +StoragePtr StorageLog::create( + const std::string & path_, + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + size_t max_compress_block_size_) { - return (new StorageLog(path_, name_, columns_, max_compress_block_size_))->thisPtr(); + return (new StorageLog{ + path_, name_, + columns_, alias_columns_, column_defaults_, + max_compress_block_size_ + })->thisPtr(); } diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 74666d65e23..05f9d275a5e 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -11,15 +11,33 @@ namespace DB { -StoragePtr StorageMaterializedView::create(const String & table_name_, const String & database_name_, - Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, bool attach_) +StoragePtr StorageMaterializedView::create( + const String & table_name_, + const String & database_name_, + Context & context_, + ASTPtr & query_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + bool attach_) { - return (new StorageMaterializedView(table_name_, database_name_, context_, query_, columns_, attach_))->thisPtr(); + return (new StorageMaterializedView{ + table_name_, database_name_, context_, query_, + columns_, alias_columns_, column_defaults_, + attach_ + })->thisPtr(); } -StorageMaterializedView::StorageMaterializedView(const String & table_name_, const String & database_name_, - Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, bool attach_): - StorageView(table_name_, database_name_, context_, query_, columns_) +StorageMaterializedView::StorageMaterializedView( + const String & table_name_, + const String & database_name_, + Context & context_, + ASTPtr & query_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + bool attach_) + : StorageView{table_name_, database_name_, context_, query_, columns_, alias_columns_, column_defaults_} { ASTCreateQuery & create = typeid_cast(*query_); diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index e790106ddc7..64c0c914dd8 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -53,14 +53,44 @@ void MemoryBlockOutputStream::write(const Block & block) } -StorageMemory::StorageMemory(const std::string & name_, NamesAndTypesListPtr columns_) +StorageMemory::StorageMemory( + const std::string & name_, + NamesAndTypesListPtr columns_) : name(name_), columns(columns_) { } -StoragePtr StorageMemory::create(const std::string & name_, NamesAndTypesListPtr columns_) + +StorageMemory::StorageMemory( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_) + : IStorage{alias_columns_, column_defaults_}, + name(name_), columns(columns_) { - return (new StorageMemory(name_, columns_))->thisPtr(); +} + + +StoragePtr StorageMemory::create( + const std::string & name_, + NamesAndTypesListPtr columns_) +{ + return (new StorageMemory{ + name_, columns_ + })->thisPtr(); +} + +StoragePtr StorageMemory::create( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_) +{ + return (new StorageMemory{ + name_, columns_, + alias_columns_, column_defaults_ + })->thisPtr(); } @@ -78,7 +108,7 @@ BlockInputStreams StorageMemory::read( Poco::ScopedLock lock(mutex); size_t size = data.size(); - + if (threads > size) threads = size; @@ -91,14 +121,14 @@ BlockInputStreams StorageMemory::read( std::advance(begin, thread * size / threads); std::advance(end, (thread + 1) * size / threads); - + res.push_back(new MemoryBlockInputStream(column_names, begin, end)); } - + return res; } - + BlockOutputStreamPtr StorageMemory::write( ASTPtr query) { diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index dadbce9e086..b1fc0668e0f 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -13,7 +13,22 @@ StorageMerge::StorageMerge( const String & source_database_, const String & table_name_regexp_, const Context & context_) - : name(name_), columns(columns_), source_database(source_database_), table_name_regexp(table_name_regexp_), context(context_) + : name(name_), columns(columns_), source_database(source_database_), + table_name_regexp(table_name_regexp_), context(context_) +{ +} + +StorageMerge::StorageMerge( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + const String & source_database_, + const String & table_name_regexp_, + const Context & context_) + : IStorage{alias_columns_, column_defaults_}, + name(name_), columns(columns_), source_database(source_database_), + table_name_regexp(table_name_regexp_), context(context_) { } @@ -24,7 +39,25 @@ StoragePtr StorageMerge::create( const String & table_name_regexp_, const Context & context_) { - return (new StorageMerge(name_, columns_, source_database_, table_name_regexp_, context_))->thisPtr(); + return (new StorageMerge{ + name_, columns_, + source_database_, table_name_regexp_, context_ + })->thisPtr(); +} + +StoragePtr StorageMerge::create( + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + const String & source_database_, + const String & table_name_regexp_, + const Context & context_) +{ + return (new StorageMerge{ + name_, columns_, alias_columns_, column_defaults_, + source_database_, table_name_regexp_, context_ + })->thisPtr(); } NameAndTypePair StorageMerge::getColumn(const String & column_name) const @@ -33,12 +66,12 @@ NameAndTypePair StorageMerge::getColumn(const String & column_name) const if (type) return NameAndTypePair(column_name, type); - return getRealColumn(column_name); + return IStorage::getColumn(column_name); } bool StorageMerge::hasColumn(const String & column_name) const { - return VirtualColumnFactory::hasColumn(column_name) || hasRealColumn(column_name); + return VirtualColumnFactory::hasColumn(column_name) || IStorage::hasColumn(column_name); } BlockInputStreams StorageMerge::read( @@ -173,4 +206,3 @@ void StorageMerge::alter(const AlterCommands & params, const String & database_n } } - diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 94fb296ef6f..bfd9307c8f4 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -7,19 +7,25 @@ namespace DB { -StorageMergeTree::StorageMergeTree(const String & path_, const String & database_name_, const String & table_name_, - NamesAndTypesListPtr columns_, - Context & context_, - ASTPtr & primary_expr_ast_, - const String & date_column_name_, - const ASTPtr & sampling_expression_, /// nullptr, если семплирование не поддерживается. - size_t index_granularity_, - MergeTreeData::Mode mode_, - const String & sign_column_, - const MergeTreeSettings & settings_) - : path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'), +StorageMergeTree::StorageMergeTree( + const String & path_, + const String & database_name_, + const String & table_name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + Context & context_, + ASTPtr & primary_expr_ast_, + const String & date_column_name_, + const ASTPtr & sampling_expression_, /// nullptr, если семплирование не поддерживается. + size_t index_granularity_, + MergeTreeData::Mode mode_, + const String & sign_column_, + const MergeTreeSettings & settings_) + : IStorage{alias_columns_, column_defaults_}, + path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'), increment(full_path + "increment.txt"), context(context_), background_pool(context_.getBackgroundPool()), - data(full_path, columns_, context_, primary_expr_ast_, date_column_name_, sampling_expression_, + data(full_path, columns_, alias_columns_, column_defaults_, context_, primary_expr_ast_, date_column_name_, sampling_expression_, index_granularity_,mode_, sign_column_, settings_, database_name_ + "." + table_name, false), reader(data), writer(data), merger(data), log(&Logger::get(database_name_ + "." + table_name + " (StorageMergeTree)")), @@ -34,6 +40,8 @@ StorageMergeTree::StorageMergeTree(const String & path_, const String & database StoragePtr StorageMergeTree::create( const String & path_, const String & database_name_, const String & table_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, Context & context_, ASTPtr & primary_expr_ast_, const String & date_column_name_, @@ -43,9 +51,12 @@ StoragePtr StorageMergeTree::create( const String & sign_column_, const MergeTreeSettings & settings_) { - StorageMergeTree * res = new StorageMergeTree( - path_, database_name_, table_name_, columns_, context_, primary_expr_ast_, date_column_name_, - sampling_expression_, index_granularity_, mode_, sign_column_, settings_); + auto res = new StorageMergeTree{ + path_, database_name_, table_name_, + columns_, alias_columns_, column_defaults_, + context_, primary_expr_ast_, date_column_name_, + sampling_expression_, index_granularity_, mode_, sign_column_, settings_ + }; StoragePtr res_ptr = res->thisPtr(); res->merge_task_handle = res->background_pool.addTask(std::bind(&StorageMergeTree::mergeTask, res, std::placeholders::_1)); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 238bd74ce71..d092e46921b 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -46,6 +46,8 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( bool attach, const String & path_, const String & database_name_, const String & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, Context & context_, ASTPtr & primary_expr_ast_, const String & date_column_name_, @@ -54,12 +56,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( MergeTreeData::Mode mode_, const String & sign_column_, const MergeTreeSettings & settings_) - : - context(context_), zookeeper(context.getZooKeeper()), database_name(database_name_), + : IStorage{alias_columns_, column_defaults_}, context(context_), + zookeeper(context.getZooKeeper()), database_name(database_name_), table_name(name_), full_path(path_ + escapeForFileName(table_name) + '/'), zookeeper_path(context.getMacros().expand(zookeeper_path_)), replica_name(context.getMacros().expand(replica_name_)), - data( full_path, columns_, context_, primary_expr_ast_, date_column_name_, sampling_expression_, + data( full_path, columns_, alias_columns_, column_defaults_, context_, + primary_expr_ast_, date_column_name_, sampling_expression_, index_granularity_, mode_, sign_column_, settings_, database_name_ + "." + table_name, true, std::bind(&StorageReplicatedMergeTree::enqueuePartForCheck, this, std::placeholders::_1)), reader(data), writer(data), merger(data), fetcher(data), @@ -113,7 +116,8 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( { LOG_INFO(log, "Have unreplicated data"); - unreplicated_data.reset(new MergeTreeData(unreplicated_path, columns_, context_, primary_expr_ast_, + unreplicated_data.reset(new MergeTreeData(unreplicated_path, columns_, alias_columns_, column_defaults_, + context_, primary_expr_ast_, date_column_name_, sampling_expression_, index_granularity_, mode_, sign_column_, settings_, database_name_ + "." + table_name + "[unreplicated]", false)); @@ -136,6 +140,8 @@ StoragePtr StorageReplicatedMergeTree::create( bool attach, const String & path_, const String & database_name_, const String & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, Context & context_, ASTPtr & primary_expr_ast_, const String & date_column_name_, @@ -145,9 +151,14 @@ StoragePtr StorageReplicatedMergeTree::create( const String & sign_column_, const MergeTreeSettings & settings_) { - StorageReplicatedMergeTree * res = new StorageReplicatedMergeTree(zookeeper_path_, replica_name_, attach, - path_, database_name_, name_, columns_, context_, primary_expr_ast_, date_column_name_, sampling_expression_, - index_granularity_, mode_, sign_column_, settings_); + auto res = new StorageReplicatedMergeTree{ + zookeeper_path_, replica_name_, attach, + path_, database_name_, name_, + columns_, alias_columns_, column_defaults_, + context_, primary_expr_ast_, date_column_name_, + sampling_expression_, index_granularity_, mode_, + sign_column_, settings_ + }; StoragePtr res_ptr = res->thisPtr(); if (!res->is_read_only) { diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 92b5a0cc65a..058cdc643c4 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -297,11 +297,18 @@ void TinyLogBlockOutputStream::write(const Block & block) } -StorageTinyLog::StorageTinyLog(const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, bool attach, size_t max_compress_block_size_) - : path(path_), name(name_), columns(columns_), - max_compress_block_size(max_compress_block_size_), - file_checker(path + escapeForFileName(name) + '/' + "sizes.json", *this), - log(&Logger::get("StorageTinyLog")) +StorageTinyLog::StorageTinyLog( + const std::string & path_, + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + bool attach, + size_t max_compress_block_size_) + : IStorage{alias_columns_, column_defaults_}, path(path_), name(name_), columns(columns_), + max_compress_block_size(max_compress_block_size_), + file_checker(path + escapeForFileName(name) + '/' + "sizes.json", *this), + log(&Logger::get("StorageTinyLog")) { if (columns->empty()) throw Exception("Empty list of columns passed to StorageTinyLog constructor", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); @@ -318,9 +325,20 @@ StorageTinyLog::StorageTinyLog(const std::string & path_, const std::string & na addFile(it->name, *it->type); } -StoragePtr StorageTinyLog::create(const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, bool attach, size_t max_compress_block_size_) +StoragePtr StorageTinyLog::create( + const std::string & path_, + const std::string & name_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + bool attach, + size_t max_compress_block_size_) { - return (new StorageTinyLog(path_, name_, columns_, attach, max_compress_block_size_))->thisPtr(); + return (new StorageTinyLog{ + path_, name_, + columns_, alias_columns_, column_defaults_, + attach, max_compress_block_size_ + })->thisPtr(); } diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 7f0f1873d98..9b24e4dd9c0 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -10,16 +10,32 @@ namespace DB { -StoragePtr StorageView::create(const String & table_name_, const String & database_name_, - Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_) +StoragePtr StorageView::create( + const String & table_name_, + const String & database_name_, + Context & context_, + ASTPtr & query_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_) { - return (new StorageView(table_name_, database_name_, context_, query_, columns_))->thisPtr(); + return (new StorageView{ + table_name_, database_name_, context_, query_, + columns_, alias_columns_, column_defaults_ + })->thisPtr(); } -StorageView::StorageView(const String & table_name_, const String & database_name_, - Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_): - table_name(table_name_), database_name(database_name_), context(context_), columns(columns_) +StorageView::StorageView( + const String & table_name_, + const String & database_name_, + Context & context_, + ASTPtr & query_, + NamesAndTypesListPtr columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_) + : IStorage{alias_columns_, column_defaults}, table_name(table_name_), + database_name(database_name_), context(context_), columns(columns_) { ASTCreateQuery & create = typeid_cast(*query_); ASTSelectQuery & select = typeid_cast(*create.select); From 34dba980c7ef453cf6375faea3636e28e993fd41 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 30 Sep 2014 07:48:40 +0400 Subject: [PATCH 08/29] show defaulted columns info in DESC TABLE query output. [#METR-12739] --- .../Interpreters/InterpreterDescribeQuery.h | 52 ++++++++++++++----- dbms/include/DB/Parsers/formatAST.h | 8 +++ .../DistributedBlockOutputStream.h | 3 +- .../DB/Storages/Distributed/queryToString.h | 14 ----- dbms/src/Storages/StorageDistributed.cpp | 1 - 5 files changed, 50 insertions(+), 28 deletions(-) delete mode 100644 dbms/include/DB/Storages/Distributed/queryToString.h diff --git a/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h b/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h index 0ff5400ea2f..a8a489ecb71 100644 --- a/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h @@ -10,6 +10,8 @@ #include #include #include +#include + namespace DB @@ -59,13 +61,18 @@ private: col.name = "name"; col.type = new DataTypeString; col.column = col.type->createColumn(); - block.insert(col); col.name = "type"; - block.insert(col); + col.name = "default_type"; + block.insert(col); + + col.name = "default_expression"; + block.insert(col); + + return block; } @@ -74,27 +81,48 @@ private: const ASTDescribeQuery & ast = typeid_cast(*query_ptr); NamesAndTypesList columns; + ColumnDefaults column_defaults; { StoragePtr table = context.getTable(ast.database, ast.table); auto table_lock = table->lockStructure(false); columns = table->getColumnsList(); + columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns)); + column_defaults = table->column_defaults; } - ColumnString * name_column = new ColumnString; - ColumnString * type_column = new ColumnString; + ColumnWithNameAndType name_column{new ColumnString, new DataTypeString, "name"}; + ColumnWithNameAndType type_column{new ColumnString, new DataTypeString, "type" }; + ColumnWithNameAndType default_type_column{new ColumnString, new DataTypeString, "default_type" }; + ColumnWithNameAndType default_expression_column{new ColumnString, new DataTypeString, "default_expression" };; - Block block; - block.insert(ColumnWithNameAndType(name_column, new DataTypeString, "name")); - block.insert(ColumnWithNameAndType(type_column, new DataTypeString, "type")); - - for (NamesAndTypesList::iterator it = columns.begin(); it != columns.end(); ++it) + auto has_defaults = false; + for (const auto column : columns) { - name_column->insert(it->name); - type_column->insert(it->type->getName()); + name_column.column->insert(column.name); + type_column.column->insert(column.type->getName()); + + const auto it = column_defaults.find(column.name); + if (it == std::end(column_defaults)) + { + default_type_column.column->insertDefault(); + default_expression_column.column->insertDefault(); + } + else + { + has_defaults = true; + default_type_column.column->insert(toString(it->second.type)); + default_expression_column.column->insert(queryToString( + setAlias(it->second.expression->clone(), ""))); + } } - return new OneBlockInputStream(block); + + return new OneBlockInputStream{ + has_defaults + ? Block{name_column, type_column, default_type_column, default_expression_column} + : Block{name_column, type_column} + }; } }; diff --git a/dbms/include/DB/Parsers/formatAST.h b/dbms/include/DB/Parsers/formatAST.h index f9f903037ca..acb0f85f83a 100644 --- a/dbms/include/DB/Parsers/formatAST.h +++ b/dbms/include/DB/Parsers/formatAST.h @@ -73,4 +73,12 @@ void formatAST(const ASTShowProcesslistQuery & ast, std::ostream & s, String formatColumnsForCreateQuery(NamesAndTypesList & columns); String backQuoteIfNeed(const String & x); +inline String queryToString(const ASTPtr & query) +{ + std::ostringstream out; + formatAST(*query, out, 0, false, true); + + return out.str(); +} + } diff --git a/dbms/include/DB/Storages/Distributed/DistributedBlockOutputStream.h b/dbms/include/DB/Storages/Distributed/DistributedBlockOutputStream.h index 1bddee6b5cc..db266cb48db 100644 --- a/dbms/include/DB/Storages/Distributed/DistributedBlockOutputStream.h +++ b/dbms/include/DB/Storages/Distributed/DistributedBlockOutputStream.h @@ -1,7 +1,8 @@ #pragma once #include -#include + +#include #include #include diff --git a/dbms/include/DB/Storages/Distributed/queryToString.h b/dbms/include/DB/Storages/Distributed/queryToString.h deleted file mode 100644 index 9f1b243ed9f..00000000000 --- a/dbms/include/DB/Storages/Distributed/queryToString.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - inline std::string queryToString(const ASTPtr & query) - { - std::ostringstream s; - formatAST(*query, s, 0, false, true); - - return s.str(); - } -} diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 242bf3b335e..da45b4f1e67 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include From 5f88c8ae81294d3f2360a356c4413904bfcc224b Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Wed, 1 Oct 2014 15:17:44 +0400 Subject: [PATCH 09/29] always show default_type and default_expression columns for DESC TABLE query. [#METR-12739] --- dbms/include/DB/Interpreters/InterpreterDescribeQuery.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h b/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h index a8a489ecb71..2f94fe854e6 100644 --- a/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h @@ -96,7 +96,6 @@ private: ColumnWithNameAndType default_type_column{new ColumnString, new DataTypeString, "default_type" }; ColumnWithNameAndType default_expression_column{new ColumnString, new DataTypeString, "default_expression" };; - auto has_defaults = false; for (const auto column : columns) { name_column.column->insert(column.name); @@ -110,18 +109,14 @@ private: } else { - has_defaults = true; default_type_column.column->insert(toString(it->second.type)); default_expression_column.column->insert(queryToString( setAlias(it->second.expression->clone(), ""))); } } - return new OneBlockInputStream{ - has_defaults - ? Block{name_column, type_column, default_type_column, default_expression_column} - : Block{name_column, type_column} + {name_column, type_column, default_type_column, default_expression_column} }; } }; From f7ce30aa9de234eb1aac5881890a5567da2bc1c8 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Fri, 3 Oct 2014 14:25:05 +0400 Subject: [PATCH 10/29] dbms: support AddingDefaultBlock(Input|Output)Stream without ColumnDefaults --- dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h | 5 +++++ .../include/DB/DataStreams/AddingDefaultBlockOutputStream.h | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h b/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h index eb9759bfa57..21648f69544 100644 --- a/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h +++ b/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h @@ -29,6 +29,11 @@ public: children.push_back(input_); } + AddingDefaultBlockInputStream(BlockInputStreamPtr input_, NamesAndTypesListPtr required_columns_, const Context & context_) + : AddingDefaultBlockInputStream{input_, required_columns, {}, context} + { + } + String getName() const { return "AddingDefaultBlockInputStream"; } String getID() const diff --git a/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h b/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h index 8089549d919..48e0450a6ea 100644 --- a/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h +++ b/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h @@ -29,6 +29,12 @@ public: { } + AddingDefaultBlockOutputStream(BlockOutputStreamPtr output_, NamesAndTypesListPtr required_columns_, const Context & context_) + : AddingDefaultBlockOutputStream{output_, required_columns_, {}, context_} + { + } + + void write(const Block & block) override { Block res = block; From 220ce78326e40ced851579ed1177b356a59bd97f Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Fri, 3 Oct 2014 19:30:10 +0400 Subject: [PATCH 11/29] dbms: support ALIAS in table declaration --- .../DB/Interpreters/ExpressionAnalyzer.h | 4 +- .../DB/Interpreters/InterpreterCreateQuery.h | 5 +- .../Interpreters/InterpreterDescribeQuery.h | 1 + dbms/include/DB/Storages/ITableDeclaration.h | 23 ++++--- .../DB/Storages/MergeTree/MergeTreeData.h | 1 + dbms/include/DB/Storages/StorageChunkMerger.h | 2 + dbms/include/DB/Storages/StorageChunks.h | 2 + dbms/include/DB/Storages/StorageDistributed.h | 2 + dbms/include/DB/Storages/StorageFactory.h | 1 + dbms/include/DB/Storages/StorageLog.h | 2 + .../DB/Storages/StorageMaterializedView.h | 2 + dbms/include/DB/Storages/StorageMemory.h | 2 + dbms/include/DB/Storages/StorageMerge.h | 2 + dbms/include/DB/Storages/StorageMergeTree.h | 2 + dbms/include/DB/Storages/StorageNull.h | 6 +- .../DB/Storages/StorageReplicatedMergeTree.h | 2 + dbms/include/DB/Storages/StorageTinyLog.h | 2 + dbms/include/DB/Storages/StorageView.h | 2 + dbms/src/Interpreters/ExpressionAnalyzer.cpp | 46 ++++++++++--- .../Interpreters/InterpreterCreateQuery.cpp | 23 ++++--- dbms/src/Storages/ITableDeclaration.cpp | 68 ++++++++++++------- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 3 +- dbms/src/Storages/StorageChunkMerger.cpp | 6 +- dbms/src/Storages/StorageChunks.cpp | 12 ++-- dbms/src/Storages/StorageDistributed.cpp | 9 ++- dbms/src/Storages/StorageFactory.cpp | 39 ++++++----- dbms/src/Storages/StorageLog.cpp | 8 ++- dbms/src/Storages/StorageMaterializedView.cpp | 8 ++- dbms/src/Storages/StorageMemory.cpp | 6 +- dbms/src/Storages/StorageMerge.cpp | 6 +- dbms/src/Storages/StorageMergeTree.cpp | 13 ++-- .../Storages/StorageReplicatedMergeTree.cpp | 19 ++++-- dbms/src/Storages/StorageSystemDatabases.cpp | 2 +- dbms/src/Storages/StorageSystemOne.cpp | 2 +- dbms/src/Storages/StorageSystemTables.cpp | 6 +- dbms/src/Storages/StorageTinyLog.cpp | 9 ++- dbms/src/Storages/StorageView.cpp | 6 +- 37 files changed, 241 insertions(+), 113 deletions(-) diff --git a/dbms/include/DB/Interpreters/ExpressionAnalyzer.h b/dbms/include/DB/Interpreters/ExpressionAnalyzer.h index 0fec5ffdea2..22aa6b900c2 100644 --- a/dbms/include/DB/Interpreters/ExpressionAnalyzer.h +++ b/dbms/include/DB/Interpreters/ExpressionAnalyzer.h @@ -210,9 +210,11 @@ private: */ void collectJoinedColumns(NameSet & joined_columns, NamesAndTypesList & joined_columns_name_type); + void addStorageAliases(); + /** Создать словарь алиасов. */ - void createAliasesDict(ASTPtr & ast, int ignore_levels = 0); + void addASTAliases(ASTPtr & ast, int ignore_levels = 0); /** Для узлов-звёздочек - раскрыть их в список всех столбцов. * Для узлов-литералов - подставить алиасы. diff --git a/dbms/include/DB/Interpreters/InterpreterCreateQuery.h b/dbms/include/DB/Interpreters/InterpreterCreateQuery.h index 7a1a5eb72e7..49a739eedbc 100644 --- a/dbms/include/DB/Interpreters/InterpreterCreateQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterCreateQuery.h @@ -27,6 +27,7 @@ public: static ASTPtr formatColumns(const NamesAndTypesList & columns); static ASTPtr formatColumns( NamesAndTypesList columns, + const NamesAndTypesList & materialized_columns, const NamesAndTypesList & alias_columns, const ColumnDefaults & column_defaults); @@ -35,8 +36,8 @@ private: using ColumnsAndDefaults = std::pair; ColumnsAndDefaults parseColumns(ASTPtr expression_list); - /// removes alias columns from the columns list and return them in a separate list - static NamesAndTypesList removeAliasColumns(ColumnsAndDefaults & columns_and_defaults); + /// removes columns from the columns list and return them in a separate list + static NamesAndTypesList removeAndReturnColumns(ColumnsAndDefaults & columns_and_defaults, ColumnDefaultType type); ASTPtr query_ptr; Context context; diff --git a/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h b/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h index 2f94fe854e6..b9a1574813f 100644 --- a/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h @@ -87,6 +87,7 @@ private: StoragePtr table = context.getTable(ast.database, ast.table); auto table_lock = table->lockStructure(false); columns = table->getColumnsList(); + columns.insert(std::end(columns), std::begin(table->materialized_columns), std::end(table->materialized_columns)); columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns)); column_defaults = table->column_defaults; } diff --git a/dbms/include/DB/Storages/ITableDeclaration.h b/dbms/include/DB/Storages/ITableDeclaration.h index 5d7f3dafe26..f796040c4e9 100644 --- a/dbms/include/DB/Storages/ITableDeclaration.h +++ b/dbms/include/DB/Storages/ITableDeclaration.h @@ -37,8 +37,8 @@ public: */ virtual bool hasRealColumn(const String & column_name) const; - NameAndTypePair getAliasColumn(const String & column_name) const; - bool hasAliasColumn(const String & column_name) const; + NameAndTypePair getMaterializedColumn(const String & column_name) const; + bool hasMaterializedColumn(const String & column_name) const; /** Получить описание любого столбца по его имени. */ @@ -57,29 +57,36 @@ public: /** Проверить, что все запрошенные имена есть в таблице и заданы корректно. * (список имён не пустой и имена не повторяются) */ - void check(const Names & column_names) const; + void check(const Names & column_names, bool all_columns = false) const; /** Проверить, что все запрошенные имена есть в таблице и имеют правильные типы. */ - void check(const NamesAndTypesList & columns) const; + void check(const NamesAndTypesList & columns, bool all_columns = false) const; /** Проверить, что все имена из пересечения names и columns есть в таблице и имеют одинаковые типы. */ - void check(const NamesAndTypesList & columns, const Names & column_names) const; + void check(const NamesAndTypesList & columns, const Names & column_names, bool all_columns = false) const; /** Проверить, что блок с данными для записи содержит все столбцы таблицы с правильными типами, * содержит только столбцы таблицы, и все столбцы различны. * Если need_all, еще проверяет, что все столбцы таблицы есть в блоке. */ - void check(const Block & block, bool need_all = false) const; + void check(const Block & block, bool need_all = false, bool all_columns = false) const; + virtual ~ITableDeclaration() = default; ITableDeclaration() = default; - ITableDeclaration(const NamesAndTypesList & alias_columns, const ColumnDefaults & column_defaults) - : alias_columns{alias_columns}, column_defaults{column_defaults} + ITableDeclaration( + const NamesAndTypesList & materialized_columns, + const NamesAndTypesList & alias_columns, + const ColumnDefaults & column_defaults) + : materialized_columns{materialized_columns}, + alias_columns{alias_columns}, + column_defaults{column_defaults} {} + NamesAndTypesList materialized_columns{}; NamesAndTypesList alias_columns{}; ColumnDefaults column_defaults{}; }; diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h index 9793b17a3fd..9ec6fbb93e0 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h @@ -583,6 +583,7 @@ public: * require_part_metadata - обязательно ли в директории с куском должны быть checksums.txt и columns.txt */ MergeTreeData( const String & full_path_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const Context & context_, diff --git a/dbms/include/DB/Storages/StorageChunkMerger.h b/dbms/include/DB/Storages/StorageChunkMerger.h index 409c42bdad5..319fbec873c 100644 --- a/dbms/include/DB/Storages/StorageChunkMerger.h +++ b/dbms/include/DB/Storages/StorageChunkMerger.h @@ -21,6 +21,7 @@ public: const std::string & this_database_,/// Имя БД для этой таблицы. const std::string & name_, /// Имя таблицы. NamesAndTypesListPtr columns_, /// Список столбцов. + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & source_database_, /// В какой БД искать таблицы-источники. @@ -74,6 +75,7 @@ private: const std::string & this_database_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & source_database_, diff --git a/dbms/include/DB/Storages/StorageChunks.h b/dbms/include/DB/Storages/StorageChunks.h index d2cf40a5070..28f2c67075b 100644 --- a/dbms/include/DB/Storages/StorageChunks.h +++ b/dbms/include/DB/Storages/StorageChunks.h @@ -23,6 +23,7 @@ public: const std::string & name_, const std::string & database_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, @@ -98,6 +99,7 @@ private: const std::string & name_, const std::string & database_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, diff --git a/dbms/include/DB/Storages/StorageDistributed.h b/dbms/include/DB/Storages/StorageDistributed.h index da4c82606be..2f3c50e2ac3 100644 --- a/dbms/include/DB/Storages/StorageDistributed.h +++ b/dbms/include/DB/Storages/StorageDistributed.h @@ -25,6 +25,7 @@ public: static StoragePtr create( const std::string & name_, /// Имя таблицы. NamesAndTypesListPtr columns_, /// Список столбцов. + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & remote_database_, /// БД на удалённых серверах. @@ -93,6 +94,7 @@ private: StorageDistributed( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & remote_database_, diff --git a/dbms/include/DB/Storages/StorageFactory.h b/dbms/include/DB/Storages/StorageFactory.h index 8510159a999..cc6aa390379 100644 --- a/dbms/include/DB/Storages/StorageFactory.h +++ b/dbms/include/DB/Storages/StorageFactory.h @@ -23,6 +23,7 @@ public: Context & context, ASTPtr & query, NamesAndTypesListPtr columns, + const NamesAndTypesList & materialized_columns, const NamesAndTypesList & alias_columns, const ColumnDefaults & column_defaults, bool attach) const; diff --git a/dbms/include/DB/Storages/StorageLog.h b/dbms/include/DB/Storages/StorageLog.h index 7a488e573b1..839863a54ac 100644 --- a/dbms/include/DB/Storages/StorageLog.h +++ b/dbms/include/DB/Storages/StorageLog.h @@ -143,6 +143,7 @@ public: const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, size_t max_compress_block_size_ = DEFAULT_MAX_COMPRESS_BLOCK_SIZE); @@ -203,6 +204,7 @@ protected: const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, size_t max_compress_block_size_); diff --git a/dbms/include/DB/Storages/StorageMaterializedView.h b/dbms/include/DB/Storages/StorageMaterializedView.h index 0add286fb49..53373b1e1cf 100644 --- a/dbms/include/DB/Storages/StorageMaterializedView.h +++ b/dbms/include/DB/Storages/StorageMaterializedView.h @@ -15,6 +15,7 @@ public: Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, bool attach_); @@ -46,6 +47,7 @@ private: Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, bool attach_); diff --git a/dbms/include/DB/Storages/StorageMemory.h b/dbms/include/DB/Storages/StorageMemory.h index bd7027c05fc..af94a302237 100644 --- a/dbms/include/DB/Storages/StorageMemory.h +++ b/dbms/include/DB/Storages/StorageMemory.h @@ -69,6 +69,7 @@ public: static StoragePtr create( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_); @@ -109,6 +110,7 @@ private: StorageMemory( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_); diff --git a/dbms/include/DB/Storages/StorageMerge.h b/dbms/include/DB/Storages/StorageMerge.h index a8d5876f05f..049f6993bec 100644 --- a/dbms/include/DB/Storages/StorageMerge.h +++ b/dbms/include/DB/Storages/StorageMerge.h @@ -28,6 +28,7 @@ public: static StoragePtr create( const std::string & name_, /// Имя таблицы. NamesAndTypesListPtr columns_, /// Список столбцов. + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & source_database_, /// В какой БД искать таблицы-источники. @@ -78,6 +79,7 @@ private: StorageMerge( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & source_database_, diff --git a/dbms/include/DB/Storages/StorageMergeTree.h b/dbms/include/DB/Storages/StorageMergeTree.h index 4c3dd9a07e8..77812c074dc 100644 --- a/dbms/include/DB/Storages/StorageMergeTree.h +++ b/dbms/include/DB/Storages/StorageMergeTree.h @@ -29,6 +29,7 @@ public: const String & database_name_, const String & table_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, @@ -163,6 +164,7 @@ private: const String & database_name_, const String & table_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, diff --git a/dbms/include/DB/Storages/StorageNull.h b/dbms/include/DB/Storages/StorageNull.h index 4d2c79f0597..4c143ac4e75 100644 --- a/dbms/include/DB/Storages/StorageNull.h +++ b/dbms/include/DB/Storages/StorageNull.h @@ -18,10 +18,11 @@ public: static StoragePtr create( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_) { - return (new StorageNull{name_, columns_, alias_columns_, column_defaults_})->thisPtr(); + return (new StorageNull{name_, columns_, materialized_columns_, alias_columns_, column_defaults_})->thisPtr(); } std::string getName() const { return "Null"; } @@ -55,9 +56,10 @@ private: StorageNull( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_) - : IStorage{alias_columns_, column_defaults_}, name(name_), columns(columns_) {} + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, name(name_), columns(columns_) {} }; } diff --git a/dbms/include/DB/Storages/StorageReplicatedMergeTree.h b/dbms/include/DB/Storages/StorageReplicatedMergeTree.h index 3c432cd20de..9ff5f754d6b 100644 --- a/dbms/include/DB/Storages/StorageReplicatedMergeTree.h +++ b/dbms/include/DB/Storages/StorageReplicatedMergeTree.h @@ -28,6 +28,7 @@ public: bool attach, const String & path_, const String & database_name_, const String & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, @@ -321,6 +322,7 @@ private: bool attach, const String & path_, const String & database_name_, const String & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, diff --git a/dbms/include/DB/Storages/StorageTinyLog.h b/dbms/include/DB/Storages/StorageTinyLog.h index 9e57f7b780b..84f23ed3e4d 100644 --- a/dbms/include/DB/Storages/StorageTinyLog.h +++ b/dbms/include/DB/Storages/StorageTinyLog.h @@ -115,6 +115,7 @@ public: const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, bool attach, @@ -170,6 +171,7 @@ private: const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, bool attach, diff --git a/dbms/include/DB/Storages/StorageView.h b/dbms/include/DB/Storages/StorageView.h index 53b6767391f..1d352ddf144 100644 --- a/dbms/include/DB/Storages/StorageView.h +++ b/dbms/include/DB/Storages/StorageView.h @@ -16,6 +16,7 @@ public: Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_); @@ -49,6 +50,7 @@ protected: Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_); }; diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index c2c61297ade..d2ae98fa7d4 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -63,8 +63,10 @@ void ExpressionAnalyzer::init() { select_query = typeid_cast(&*ast); + addStorageAliases(); + /// Создаёт словарь aliases: alias -> ASTPtr - createAliasesDict(ast); + addASTAliases(ast); /// Common subexpression elimination. Rewrite rules. normalizeTree(); @@ -235,9 +237,19 @@ NamesAndTypesList::iterator ExpressionAnalyzer::findColumn(const String & name, } +void ExpressionAnalyzer::addStorageAliases() +{ + if (!storage) + return; + + for (const auto & alias : storage->alias_columns) + aliases[alias.name] = storage->column_defaults[alias.name].expression; +} + + /// ignore_levels - алиасы в скольки верхних уровнях поддерева нужно игнорировать. /// Например, при ignore_levels=1 ast не может быть занесен в словарь, но его дети могут. -void ExpressionAnalyzer::createAliasesDict(ASTPtr & ast, int ignore_levels) +void ExpressionAnalyzer::addASTAliases(ASTPtr & ast, int ignore_levels) { ASTSelectQuery * select = typeid_cast(&*ast); @@ -252,7 +264,7 @@ void ExpressionAnalyzer::createAliasesDict(ASTPtr & ast, int ignore_levels) new_ignore_levels = 2; if (!typeid_cast(&*child)) - createAliasesDict(child, new_ignore_levels); + addASTAliases(child, new_ignore_levels); } if (ignore_levels > 0) @@ -1683,18 +1695,34 @@ void ExpressionAnalyzer::collectUsedColumns() ++it; } - /// Возможно, среди неизвестных столбцов есть виртуальные. Удаляем их из списка неизвестных и добавляем - /// в columns list, чтобы при дальнейшей обработке запроса они воспринимались как настоящие. - for (NameSet::iterator it = unknown_required_columns.begin(); it != unknown_required_columns.end();) + for (NamesAndTypesList::iterator it = columns.begin(); it != columns.end();) { - if (storage && storage->hasColumn(*it)) + unknown_required_columns.erase(it->name); + + if (!required.count(it->name)) { - columns.push_back(storage->getColumn(*it)); - unknown_required_columns.erase(it++); + required.erase(it->name); + columns.erase(it++); } else ++it; } + + /// Возможно, среди неизвестных столбцов есть виртуальные. Удаляем их из списка неизвестных и добавляем + /// в columns list, чтобы при дальнейшей обработке запроса они воспринимались как настоящие. + if (storage) + { + for (auto it = unknown_required_columns.begin(); it != unknown_required_columns.end();) + { + if (storage->hasColumn(*it)) + { + columns.push_back(storage->getColumn(*it)); + unknown_required_columns.erase(it++); + } + else + ++it; + } + } } void ExpressionAnalyzer::collectJoinedColumns(NameSet & joined_columns, NamesAndTypesList & joined_columns_name_type) diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 46b03c7cba1..2577ebdc51a 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -89,6 +89,7 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) StoragePtr res; String storage_name; NamesAndTypesListPtr columns = new NamesAndTypesList; + NamesAndTypesList materialized_columns{}; NamesAndTypesList alias_columns{}; ColumnDefaults column_defaults{}; @@ -120,7 +121,8 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) if (create.columns) { auto && columns_and_defaults = parseColumns(create.columns); - alias_columns = removeAliasColumns(columns_and_defaults); + materialized_columns = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Materialized); + alias_columns = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Alias); columns = new NamesAndTypesList{std::move(columns_and_defaults.first)}; column_defaults = std::move(columns_and_defaults.second); } @@ -136,7 +138,7 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY); /// Даже если в запросе был список столбцов, на всякий случай приведем его к стандартному виду (развернем Nested). - ASTPtr new_columns = formatColumns(*columns, alias_columns, column_defaults); + ASTPtr new_columns = formatColumns(*columns, materialized_columns, alias_columns, column_defaults); if (create.columns) { auto it = std::find(create.children.begin(), create.children.end(), create.columns); @@ -185,8 +187,8 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) res = context.getStorageFactory().get( storage_name, data_path, table_name, database_name, context, - context.getGlobalContext(), query_ptr, - columns, alias_columns, column_defaults, create.attach); + context.getGlobalContext(), query_ptr, columns, + materialized_columns, alias_columns, column_defaults, create.attach); /// Проверка наличия метаданных таблицы на диске и создание метаданных if (!assume_metadata_exists && !create.is_temporary) @@ -342,26 +344,27 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( return { *DataTypeNested::expandNestedColumns(columns), defaults }; } -NamesAndTypesList InterpreterCreateQuery::removeAliasColumns(ColumnsAndDefaults & columns_and_defaults) +NamesAndTypesList InterpreterCreateQuery::removeAndReturnColumns(ColumnsAndDefaults & columns_and_defaults, + const ColumnDefaultType type) { auto & columns = columns_and_defaults.first; auto & defaults = columns_and_defaults.second; - NamesAndTypesList alias_columns{}; + NamesAndTypesList removed{}; for (auto it = std::begin(columns); it != std::end(columns);) { const auto jt = defaults.find(it->name); - if (jt != std::end(defaults) && jt->second.type != ColumnDefaultType::Default) + if (jt != std::end(defaults) && jt->second.type == type) { - alias_columns.push_back(*it); + removed.push_back(*it); it = columns.erase(it); } else ++it; } - return alias_columns; + return removed; } ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns) @@ -393,9 +396,11 @@ ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns) } ASTPtr InterpreterCreateQuery::formatColumns(NamesAndTypesList columns, + const NamesAndTypesList & materialized_columns, const NamesAndTypesList & alias_columns, const ColumnDefaults & column_defaults) { + columns.insert(std::end(columns), std::begin(materialized_columns), std::end(materialized_columns)); columns.insert(std::end(columns), std::begin(alias_columns), std::end(alias_columns)); ASTPtr columns_list_ptr{new ASTExpressionList}; diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index c81fb242513..c9383f5e3d9 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -40,18 +40,18 @@ NameAndTypePair ITableDeclaration::getRealColumn(const String & column_name) con throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); } -NameAndTypePair ITableDeclaration::getAliasColumn(const String & column_name) const +NameAndTypePair ITableDeclaration::getMaterializedColumn(const String & column_name) const { - for (auto & column : alias_columns) + for (auto & column : materialized_columns) if (column.name == column_name) return column; throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); } -bool ITableDeclaration::hasAliasColumn(const String & column_name) const +bool ITableDeclaration::hasMaterializedColumn(const String & column_name) const { - for (auto & column : alias_columns) + for (auto & column : materialized_columns) if (column.name == column_name) return true; @@ -60,7 +60,7 @@ bool ITableDeclaration::hasAliasColumn(const String & column_name) const bool ITableDeclaration::hasColumn(const String & column_name) const { - return hasRealColumn(column_name) || hasAliasColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. + return hasRealColumn(column_name) || hasMaterializedColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. } NameAndTypePair ITableDeclaration::getColumn(const String & column_name) const @@ -70,16 +70,19 @@ NameAndTypePair ITableDeclaration::getColumn(const String & column_name) const if (it == std::end(column_defaults) || it->second.type == ColumnDefaultType::Default) return getRealColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. - return getAliasColumn(column_name); + return getMaterializedColumn(column_name); } const DataTypePtr ITableDeclaration::getDataTypeByName(const String & column_name) const { - const NamesAndTypesList & names_and_types = getColumnsList(); - for (NamesAndTypesList::const_iterator it = names_and_types.begin(); it != names_and_types.end(); ++it) - if (it->name == column_name) - return it->type; + for (const auto & column : getColumnsList()) + if (column.name == column_name) + return column.type; + + for (const auto & column : materialized_columns) + if (column.name == column_name) + return column.type; throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); } @@ -118,20 +121,30 @@ static std::string listOfColumns(const NamesAndTypesList & available_columns) typedef google::dense_hash_map NamesAndTypesMap; +static NamesAndTypesMap & getColumnsMapImpl(NamesAndTypesMap & res) { return res; } -static NamesAndTypesMap getColumnsMap(const NamesAndTypesList & available_columns) +template +static NamesAndTypesMap & getColumnsMapImpl(NamesAndTypesMap & res, const Arg & arg, const Args &... args) +{ + static_assert(std::is_same::value, "getColumnsMap requires arguments of type NamesAndTypesList"); + + for (const auto & column : arg) + res.insert({column.name, column.type.get()}); + + return getColumnsMapImpl(res, args...); +} + +template +static NamesAndTypesMap getColumnsMap(const Args &... args) { NamesAndTypesMap res; res.set_empty_key(StringRef()); - for (NamesAndTypesList::const_iterator it = available_columns.begin(); it != available_columns.end(); ++it) - res.insert(NamesAndTypesMap::value_type(it->name, &*it->type)); - - return res; + return getColumnsMapImpl(res, args...); } -void ITableDeclaration::check(const Names & column_names) const +void ITableDeclaration::check(const Names & column_names, const bool all_columns) const { const NamesAndTypesList & available_columns = getColumnsList(); @@ -139,7 +152,9 @@ void ITableDeclaration::check(const Names & column_names) const throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns), ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED); - const NamesAndTypesMap & columns_map = getColumnsMap(available_columns); + const auto columns_map = all_columns ? + getColumnsMap(available_columns, materialized_columns) : + getColumnsMap(available_columns); typedef google::dense_hash_set UniqueStrings; UniqueStrings unique_names; @@ -159,10 +174,12 @@ void ITableDeclaration::check(const Names & column_names) const } -void ITableDeclaration::check(const NamesAndTypesList & columns) const +void ITableDeclaration::check(const NamesAndTypesList & columns, const bool all_columns) const { const NamesAndTypesList & available_columns = getColumnsList(); - const NamesAndTypesMap & columns_map = getColumnsMap(available_columns); + const auto columns_map = all_columns ? + getColumnsMap(available_columns, materialized_columns) : + getColumnsMap(available_columns); typedef google::dense_hash_set UniqueStrings; UniqueStrings unique_names; @@ -187,10 +204,12 @@ void ITableDeclaration::check(const NamesAndTypesList & columns) const } -void ITableDeclaration::check(const NamesAndTypesList & columns, const Names & column_names) const +void ITableDeclaration::check(const NamesAndTypesList & columns, const Names & column_names, const bool all_columns) const { const NamesAndTypesList & available_columns = getColumnsList(); - const NamesAndTypesMap & available_columns_map = getColumnsMap(available_columns); + const auto available_columns_map = all_columns ? + getColumnsMap(available_columns, materialized_columns) : + getColumnsMap(available_columns); const NamesAndTypesMap & provided_columns_map = getColumnsMap(columns); if (column_names.empty()) @@ -224,10 +243,13 @@ void ITableDeclaration::check(const NamesAndTypesList & columns, const Names & c } -void ITableDeclaration::check(const Block & block, bool need_all) const +void ITableDeclaration::check(const Block & block, bool need_all, const bool all_columns) const { const NamesAndTypesList & available_columns = getColumnsList(); - const NamesAndTypesMap & columns_map = getColumnsMap(available_columns); + const auto columns_map = all_columns ? + getColumnsMap(available_columns, materialized_columns) : + getColumnsMap(available_columns); + typedef std::unordered_set NameSet; NameSet names_in_block; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 6cefc2393e1..ba3eb9e25ee 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -21,6 +21,7 @@ namespace DB MergeTreeData::MergeTreeData( const String & full_path_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const Context & context_, @@ -33,7 +34,7 @@ MergeTreeData::MergeTreeData( const String & log_name_, bool require_part_metadata_, BrokenPartCallback broken_part_callback_) - : ITableDeclaration{alias_columns_, column_defaults_}, context(context_), + : ITableDeclaration{materialized_columns_, alias_columns_, column_defaults_}, context(context_), date_column_name(date_column_name_), sampling_expression(sampling_expression_), index_granularity(index_granularity_), mode(mode_), sign_column(sign_column_), diff --git a/dbms/src/Storages/StorageChunkMerger.cpp b/dbms/src/Storages/StorageChunkMerger.cpp index 02014503be2..ba4ecf96924 100644 --- a/dbms/src/Storages/StorageChunkMerger.cpp +++ b/dbms/src/Storages/StorageChunkMerger.cpp @@ -30,6 +30,7 @@ StoragePtr StorageChunkMerger::create( const std::string & this_database_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & source_database_, @@ -39,7 +40,7 @@ StoragePtr StorageChunkMerger::create( Context & context_) { return (new StorageChunkMerger{ - this_database_, name_, columns_, alias_columns_, column_defaults_, + this_database_, name_, columns_, materialized_columns_, alias_columns_, column_defaults_, source_database_, table_name_regexp_, destination_name_prefix_, chunks_to_merge_, context_ })->thisPtr(); @@ -227,6 +228,7 @@ StorageChunkMerger::StorageChunkMerger( const std::string & this_database_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & source_database_, @@ -234,7 +236,7 @@ StorageChunkMerger::StorageChunkMerger( const std::string & destination_name_prefix_, size_t chunks_to_merge_, Context & context_) - : IStorage{alias_columns_, column_defaults_}, + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, this_database(this_database_), name(name_), columns(columns_), source_database(source_database_), table_name_regexp(table_name_regexp_), destination_name_prefix(destination_name_prefix_), chunks_to_merge(chunks_to_merge_), context(context_), settings(context.getSettings()), diff --git a/dbms/src/Storages/StorageChunks.cpp b/dbms/src/Storages/StorageChunks.cpp index 05b9b6e376e..8ebb0605754 100644 --- a/dbms/src/Storages/StorageChunks.cpp +++ b/dbms/src/Storages/StorageChunks.cpp @@ -16,14 +16,15 @@ StoragePtr StorageChunks::create( const std::string & name_, const std::string & database_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, bool attach) { return (new StorageChunks{ - path_, name_, database_name_, - columns_, alias_columns_, column_defaults_, + path_, name_, database_name_, columns_, + materialized_columns_, alias_columns_, column_defaults_, context_, attach })->thisPtr(); } @@ -136,12 +137,15 @@ StorageChunks::StorageChunks( const std::string & name_, const std::string & database_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, bool attach) : - StorageLog(path_, name_, columns_, alias_columns_, column_defaults_, context_.getSettings().max_compress_block_size), + StorageLog(path_, name_, columns_, + materialized_columns_, alias_columns_, column_defaults_, + context_.getSettings().max_compress_block_size), database_name(database_name_), reference_counter(path_ + escapeForFileName(name_) + "/refcount.txt"), context(context_), @@ -179,7 +183,7 @@ NameAndTypePair StorageChunks::getColumn(const String &column_name) const bool StorageChunks::hasColumn(const String &column_name) const { if (column_name == _table_column_name) return true; - return hasRealColumn(column_name); + return IStorage::hasColumn(column_name); } std::pair StorageChunks::getTableFromMark(size_t mark) const diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index da45b4f1e67..98d0785907e 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -72,6 +72,7 @@ StorageDistributed::StorageDistributed( StorageDistributed::StorageDistributed( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & remote_database_, @@ -80,7 +81,7 @@ StorageDistributed::StorageDistributed( Context & context_, const ASTPtr & sharding_key_, const String & data_path_) - : IStorage{alias_columns_, column_defaults_}, + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, name(name_), columns(columns_), remote_database(remote_database_), remote_table(remote_table_), context(context_), cluster(cluster_), @@ -95,6 +96,7 @@ StorageDistributed::StorageDistributed( StoragePtr StorageDistributed::create( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & remote_database_, @@ -107,7 +109,8 @@ StoragePtr StorageDistributed::create( context_.initClusters(); return (new StorageDistributed{ - name_, columns_, alias_columns_, column_defaults_, + name_, columns_, + materialized_columns_, alias_columns_, column_defaults_, remote_database_, remote_table_, context_.getCluster(cluster_name), context_, sharding_key_, data_path_ @@ -219,7 +222,7 @@ NameAndTypePair StorageDistributed::getColumn(const String & column_name) const bool StorageDistributed::hasColumn(const String & column_name) const { - return VirtualColumnFactory::hasColumn(column_name) || hasRealColumn(column_name); + return VirtualColumnFactory::hasColumn(column_name) || IStorage::hasColumn(column_name); } void StorageDistributed::createDirectoryMonitor(const std::string & name) diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 507e5853e50..f8183db284d 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -72,6 +72,7 @@ StoragePtr StorageFactory::get( Context & context, ASTPtr & query, NamesAndTypesListPtr columns, + const NamesAndTypesList & materialized_columns, const NamesAndTypesList & alias_columns, const ColumnDefaults & column_defaults, bool attach) const @@ -79,15 +80,15 @@ StoragePtr StorageFactory::get( if (name == "Log") { return StorageLog::create( - data_path, table_name, - columns, alias_columns, column_defaults, + data_path, table_name, columns, + materialized_columns, alias_columns, column_defaults, context.getSettings().max_compress_block_size); } else if (name == "Chunks") { return StorageChunks::create( - data_path, table_name, database_name, - columns, alias_columns, column_defaults, + data_path, table_name, database_name, columns, + materialized_columns, alias_columns, column_defaults, context, attach); } else if (name == "ChunkRef") @@ -97,14 +98,14 @@ StoragePtr StorageFactory::get( else if (name == "View") { return StorageView::create( - table_name, database_name, context, query, - columns, alias_columns, column_defaults); + table_name, database_name, context, query, columns, + materialized_columns, alias_columns, column_defaults); } else if (name == "MaterializedView") { return StorageMaterializedView::create( - table_name, database_name, context, query, - columns, alias_columns, column_defaults, + table_name, database_name, context, query, columns, + materialized_columns, alias_columns, column_defaults, attach); } else if (name == "ChunkMerger") @@ -132,8 +133,8 @@ StoragePtr StorageFactory::get( destination_name_prefix = typeid_cast(*args[3]).name; return StorageChunkMerger::create( - database_name, table_name, - columns, alias_columns, column_defaults, + database_name, table_name, columns, + materialized_columns, alias_columns, column_defaults, source_database, source_table_name_regexp, destination_name_prefix, chunks_to_merge, context); } while (false); @@ -145,17 +146,17 @@ StoragePtr StorageFactory::get( else if (name == "TinyLog") { return StorageTinyLog::create( - data_path, table_name, - columns, alias_columns, column_defaults, + data_path, table_name, columns, + materialized_columns, alias_columns, column_defaults, attach, context.getSettings().max_compress_block_size); } else if (name == "Memory") { - return StorageMemory::create(table_name, columns, alias_columns, column_defaults); + return StorageMemory::create(table_name, columns, materialized_columns, alias_columns, column_defaults); } else if (name == "Null") { - return StorageNull::create(table_name, columns, alias_columns, column_defaults); + return StorageNull::create(table_name, columns, materialized_columns, alias_columns, column_defaults); } else if (name == "Merge") { @@ -180,7 +181,8 @@ StoragePtr StorageFactory::get( String table_name_regexp = safeGet(typeid_cast(*args[1]).value); return StorageMerge::create( - table_name, columns, alias_columns, column_defaults, + table_name, columns, + materialized_columns, alias_columns, column_defaults, source_database, table_name_regexp, context); } else if (name == "Distributed") @@ -210,7 +212,8 @@ StoragePtr StorageFactory::get( const auto & sharding_key = args.size() == 4 ? args[3] : nullptr; return StorageDistributed::create( - table_name, columns, alias_columns, column_defaults, + table_name, columns, + materialized_columns, alias_columns, column_defaults, remote_database, remote_table, cluster_name, context, sharding_key, data_path); } @@ -328,13 +331,13 @@ StoragePtr StorageFactory::get( if (replicated) return StorageReplicatedMergeTree::create( zookeeper_path, replica_name, attach, data_path, database_name, table_name, - columns, alias_columns, column_defaults, + columns, materialized_columns, alias_columns, column_defaults, context, primary_expr_list, date_column_name, sampling_expression, index_granularity, mode, sign_column_name); else return StorageMergeTree::create( data_path, database_name, table_name, - columns, alias_columns, column_defaults, + columns, materialized_columns, alias_columns, column_defaults, context, primary_expr_list, date_column_name, sampling_expression, index_granularity, mode, sign_column_name); } diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index ccd26e17b20..26e303e72b2 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -413,10 +413,11 @@ StorageLog::StorageLog( const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, size_t max_compress_block_size_) - : IStorage{alias_columns_, column_defaults_}, + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, path(path_), name(name_), columns(columns_), loaded_marks(false), max_compress_block_size(max_compress_block_size_), file_checker(path + escapeForFileName(name) + '/' + "sizes.json", *this) @@ -437,13 +438,14 @@ StoragePtr StorageLog::create( const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, size_t max_compress_block_size_) { return (new StorageLog{ - path_, name_, - columns_, alias_columns_, column_defaults_, + path_, name_, columns_, + materialized_columns_, alias_columns_, column_defaults_, max_compress_block_size_ })->thisPtr(); } diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 05f9d275a5e..a9560808b1d 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -17,13 +17,14 @@ StoragePtr StorageMaterializedView::create( Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, bool attach_) { return (new StorageMaterializedView{ table_name_, database_name_, context_, query_, - columns_, alias_columns_, column_defaults_, + columns_, materialized_columns_, alias_columns_, column_defaults_, attach_ })->thisPtr(); } @@ -34,10 +35,11 @@ StorageMaterializedView::StorageMaterializedView( Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, bool attach_) - : StorageView{table_name_, database_name_, context_, query_, columns_, alias_columns_, column_defaults_} + : StorageView{table_name_, database_name_, context_, query_, columns_, materialized_columns_, alias_columns_, column_defaults_} { ASTCreateQuery & create = typeid_cast(*query_); @@ -91,7 +93,7 @@ NameAndTypePair StorageMaterializedView::getColumn(const String & column_name) c bool StorageMaterializedView::hasColumn(const String & column_name) const { - return VirtualColumnFactory::hasColumn(column_name) || hasRealColumn(column_name); + return VirtualColumnFactory::hasColumn(column_name) || IStorage::hasColumn(column_name); } BlockInputStreams StorageMaterializedView::read( diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index 64c0c914dd8..b5ba1dac28d 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -64,9 +64,10 @@ StorageMemory::StorageMemory( StorageMemory::StorageMemory( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_) - : IStorage{alias_columns_, column_defaults_}, + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, name(name_), columns(columns_) { } @@ -84,12 +85,13 @@ StoragePtr StorageMemory::create( StoragePtr StorageMemory::create( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_) { return (new StorageMemory{ name_, columns_, - alias_columns_, column_defaults_ + materialized_columns_, alias_columns_, column_defaults_ })->thisPtr(); } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index b1fc0668e0f..13409135104 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -21,12 +21,13 @@ StorageMerge::StorageMerge( StorageMerge::StorageMerge( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & source_database_, const String & table_name_regexp_, const Context & context_) - : IStorage{alias_columns_, column_defaults_}, + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, name(name_), columns(columns_), source_database(source_database_), table_name_regexp(table_name_regexp_), context(context_) { @@ -48,6 +49,7 @@ StoragePtr StorageMerge::create( StoragePtr StorageMerge::create( const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, const String & source_database_, @@ -55,7 +57,7 @@ StoragePtr StorageMerge::create( const Context & context_) { return (new StorageMerge{ - name_, columns_, alias_columns_, column_defaults_, + name_, columns_, materialized_columns_, alias_columns_, column_defaults_, source_database_, table_name_regexp_, context_ })->thisPtr(); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 7c3a5bc2380..9807e1738f8 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -12,6 +12,7 @@ StorageMergeTree::StorageMergeTree( const String & database_name_, const String & table_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, @@ -22,11 +23,14 @@ StorageMergeTree::StorageMergeTree( MergeTreeData::Mode mode_, const String & sign_column_, const MergeTreeSettings & settings_) - : IStorage{alias_columns_, column_defaults_}, + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'), increment(full_path + "increment.txt"), context(context_), background_pool(context_.getBackgroundPool()), - data(full_path, columns_, alias_columns_, column_defaults_, context_, primary_expr_ast_, date_column_name_, sampling_expression_, - index_granularity_,mode_, sign_column_, settings_, database_name_ + "." + table_name, false), + data(full_path, columns_, + materialized_columns_, alias_columns_, column_defaults_, + context_, primary_expr_ast_, date_column_name_, + sampling_expression_, index_granularity_,mode_, sign_column_, + settings_, database_name_ + "." + table_name, false), reader(data), writer(data), merger(data), log(&Logger::get(database_name_ + "." + table_name + " (StorageMergeTree)")), shutdown_called(false) @@ -40,6 +44,7 @@ StorageMergeTree::StorageMergeTree( StoragePtr StorageMergeTree::create( const String & path_, const String & database_name_, const String & table_name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, @@ -53,7 +58,7 @@ StoragePtr StorageMergeTree::create( { auto res = new StorageMergeTree{ path_, database_name_, table_name_, - columns_, alias_columns_, column_defaults_, + columns_, materialized_columns_, alias_columns_, column_defaults_, context_, primary_expr_ast_, date_column_name_, sampling_expression_, index_granularity_, mode_, sign_column_, settings_ }; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 95525bd71bb..088f302b57d 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -46,6 +46,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( bool attach, const String & path_, const String & database_name_, const String & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, @@ -56,15 +57,17 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( MergeTreeData::Mode mode_, const String & sign_column_, const MergeTreeSettings & settings_) - : IStorage{alias_columns_, column_defaults_}, context(context_), + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, context(context_), zookeeper(context.getZooKeeper()), database_name(database_name_), table_name(name_), full_path(path_ + escapeForFileName(table_name) + '/'), zookeeper_path(context.getMacros().expand(zookeeper_path_)), replica_name(context.getMacros().expand(replica_name_)), - data( full_path, columns_, alias_columns_, column_defaults_, context_, - primary_expr_ast_, date_column_name_, sampling_expression_, - index_granularity_, mode_, sign_column_, settings_, database_name_ + "." + table_name, true, - std::bind(&StorageReplicatedMergeTree::enqueuePartForCheck, this, std::placeholders::_1)), + data(full_path, columns_, + materialized_columns_, alias_columns_, column_defaults_, + context_, primary_expr_ast_, date_column_name_, + sampling_expression_, index_granularity_, mode_, sign_column_, + settings_, database_name_ + "." + table_name, true, + std::bind(&StorageReplicatedMergeTree::enqueuePartForCheck, this, std::placeholders::_1)), reader(data), writer(data), merger(data), fetcher(data), log(&Logger::get(database_name_ + "." + table_name + " (StorageReplicatedMergeTree)")), shutdown_event(false) @@ -116,7 +119,8 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( { LOG_INFO(log, "Have unreplicated data"); - unreplicated_data.reset(new MergeTreeData(unreplicated_path, columns_, alias_columns_, column_defaults_, + unreplicated_data.reset(new MergeTreeData(unreplicated_path, columns_, + materialized_columns_, alias_columns_, column_defaults_, context_, primary_expr_ast_, date_column_name_, sampling_expression_, index_granularity_, mode_, sign_column_, settings_, database_name_ + "." + table_name + "[unreplicated]", false)); @@ -140,6 +144,7 @@ StoragePtr StorageReplicatedMergeTree::create( bool attach, const String & path_, const String & database_name_, const String & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, Context & context_, @@ -154,7 +159,7 @@ StoragePtr StorageReplicatedMergeTree::create( auto res = new StorageReplicatedMergeTree{ zookeeper_path_, replica_name_, attach, path_, database_name_, name_, - columns_, alias_columns_, column_defaults_, + columns_, materialized_columns_, alias_columns_, column_defaults_, context_, primary_expr_ast_, date_column_name_, sampling_expression_, index_granularity_, mode_, sign_column_, settings_ diff --git a/dbms/src/Storages/StorageSystemDatabases.cpp b/dbms/src/Storages/StorageSystemDatabases.cpp index 063f4637394..0a27756a1a1 100644 --- a/dbms/src/Storages/StorageSystemDatabases.cpp +++ b/dbms/src/Storages/StorageSystemDatabases.cpp @@ -28,7 +28,7 @@ BlockInputStreams StorageSystemDatabases::read( processed_stage = QueryProcessingStage::FetchColumns; Block block; - + ColumnWithNameAndType col_name; col_name.name = "name"; col_name.type = new DataTypeString; diff --git a/dbms/src/Storages/StorageSystemOne.cpp b/dbms/src/Storages/StorageSystemOne.cpp index de12e93fa0b..b600a193ec3 100644 --- a/dbms/src/Storages/StorageSystemOne.cpp +++ b/dbms/src/Storages/StorageSystemOne.cpp @@ -36,7 +36,7 @@ BlockInputStreams StorageSystemOne::read( col.type = new DataTypeUInt8; col.column = new ColumnConstUInt8(1, 0); block.insert(col); - + return BlockInputStreams(1, new OneBlockInputStream(block)); } diff --git a/dbms/src/Storages/StorageSystemTables.cpp b/dbms/src/Storages/StorageSystemTables.cpp index 6af90136564..68acc504df7 100644 --- a/dbms/src/Storages/StorageSystemTables.cpp +++ b/dbms/src/Storages/StorageSystemTables.cpp @@ -30,7 +30,7 @@ BlockInputStreams StorageSystemTables::read( processed_stage = QueryProcessingStage::FetchColumns; Block block; - + ColumnWithNameAndType col_db; col_db.name = "database"; col_db.type = new DataTypeString; @@ -50,7 +50,7 @@ BlockInputStreams StorageSystemTables::read( block.insert(col_engine); Poco::ScopedLock lock(context.getMutex()); - + for (Databases::const_iterator it = context.getDatabases().begin(); it != context.getDatabases().end(); ++it) { for (Tables::const_iterator jt = it->second.begin(); jt != it->second.end(); ++jt) @@ -60,7 +60,7 @@ BlockInputStreams StorageSystemTables::read( col_engine.column->insert(jt->second->getName()); } } - + return BlockInputStreams(1, new OneBlockInputStream(block)); } diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 058cdc643c4..faf19a0d695 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -301,11 +301,13 @@ StorageTinyLog::StorageTinyLog( const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, bool attach, size_t max_compress_block_size_) - : IStorage{alias_columns_, column_defaults_}, path(path_), name(name_), columns(columns_), + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, + path(path_), name(name_), columns(columns_), max_compress_block_size(max_compress_block_size_), file_checker(path + escapeForFileName(name) + '/' + "sizes.json", *this), log(&Logger::get("StorageTinyLog")) @@ -329,14 +331,15 @@ StoragePtr StorageTinyLog::create( const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_, bool attach, size_t max_compress_block_size_) { return (new StorageTinyLog{ - path_, name_, - columns_, alias_columns_, column_defaults_, + path_, name_, columns_, + materialized_columns_, alias_columns_, column_defaults_, attach, max_compress_block_size_ })->thisPtr(); } diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 9b24e4dd9c0..156385a61c1 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -16,12 +16,13 @@ StoragePtr StorageView::create( Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_) { return (new StorageView{ table_name_, database_name_, context_, query_, - columns_, alias_columns_, column_defaults_ + columns_, materialized_columns_, alias_columns_, column_defaults_ })->thisPtr(); } @@ -32,9 +33,10 @@ StorageView::StorageView( Context & context_, ASTPtr & query_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_) - : IStorage{alias_columns_, column_defaults}, table_name(table_name_), + : IStorage{materialized_columns_, alias_columns_, column_defaults}, table_name(table_name_), database_name(database_name_), context(context_), columns(columns_) { ASTCreateQuery & create = typeid_cast(*query_); From f675bed75b79885e84e2efe05008a4b1591ab382 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 7 Oct 2014 13:09:59 +0400 Subject: [PATCH 12/29] dbms: add partial validation of ALTER queries. [#METR-12739] --- .../DB/Interpreters/InterpreterAlterQuery.h | 4 +- dbms/include/DB/Parsers/ASTAlterQuery.h | 8 +- .../include/DB/Parsers/ASTColumnDeclaration.h | 1 - dbms/include/DB/Parsers/ParserCreateQuery.h | 9 +- dbms/include/DB/Storages/AlterCommands.h | 9 +- .../Interpreters/InterpreterAlterQuery.cpp | 157 ++++++++++++++++-- .../Interpreters/InterpreterCreateQuery.cpp | 4 - dbms/src/Parsers/ParserAlterQuery.cpp | 6 +- dbms/src/Parsers/formatAST.cpp | 4 +- 9 files changed, 169 insertions(+), 33 deletions(-) diff --git a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h index e3f44a0efd8..9da89aaccb7 100644 --- a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h @@ -53,10 +53,12 @@ private: typedef std::vector PartitionCommands; ASTPtr query_ptr; - + Context context; static void parseAlter(const ASTAlterQuery::ParameterContainer & params, const DataTypeFactory & data_type_factory, AlterCommands & out_alter_commands, PartitionCommands & out_partition_commands); + + static void validateColumnChanges(ASTAlterQuery::ParameterContainer & params, const StoragePtr & table, const Context & context); }; } diff --git a/dbms/include/DB/Parsers/ASTAlterQuery.h b/dbms/include/DB/Parsers/ASTAlterQuery.h index 897db72a0b9..be80b454256 100644 --- a/dbms/include/DB/Parsers/ASTAlterQuery.h +++ b/dbms/include/DB/Parsers/ASTAlterQuery.h @@ -36,7 +36,7 @@ public: * В запросе DROP это поле не используется * В запросе MODIFY здесь хранится имя столбца и новый тип */ - ASTPtr name_type; + ASTPtr col_decl; /** В запросе ADD COLUMN здесь опционально хранится имя столбца, следующее после AFTER * В запросе DROP здесь хранится имя столбца для удаления @@ -55,7 +55,7 @@ public: void clone(Parameters & p) const { p = *this; - p.name_type = name_type->clone(); + p.col_decl = col_decl->clone(); p.column = column->clone(); p.partition = partition->clone(); } @@ -69,8 +69,8 @@ public: void addParameters(const Parameters & params) { parameters.push_back(params); - if (params.name_type) - children.push_back(params.name_type); + if (params.col_decl) + children.push_back(params.col_decl); if (params.column) children.push_back(params.column); if (params.partition) diff --git a/dbms/include/DB/Parsers/ASTColumnDeclaration.h b/dbms/include/DB/Parsers/ASTColumnDeclaration.h index a4f017586d6..13fb158955b 100644 --- a/dbms/include/DB/Parsers/ASTColumnDeclaration.h +++ b/dbms/include/DB/Parsers/ASTColumnDeclaration.h @@ -41,4 +41,3 @@ public: }; } - diff --git a/dbms/include/DB/Parsers/ParserCreateQuery.h b/dbms/include/DB/Parsers/ParserCreateQuery.h index ce6a51ae145..d693ee470ad 100644 --- a/dbms/include/DB/Parsers/ParserCreateQuery.h +++ b/dbms/include/DB/Parsers/ParserCreateQuery.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -112,9 +113,9 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, Pos end, ASTPtr NameParser name_parser; ParserIdentifierWithOptionalParameters type_parser; ParserWhiteSpaceOrComments ws; - ParserString s_default{"DEFAULT"}; - ParserString s_materialized{"MATERIALIZED"}; - ParserString s_alias{"ALIAS"}; + ParserString s_default{"DEFAULT", true, true}; + ParserString s_materialized{"MATERIALIZED", true, true}; + ParserString s_alias{"ALIAS", true, true}; ParserTernaryOperatorExpression expr_parser; const auto begin = pos; @@ -152,7 +153,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, Pos end, ASTPtr s_materialized.ignore(pos, end, expected) || s_alias.ignore(pos, end, expected)) { - default_specifier.assign(pos_before_specifier, pos); + default_specifier = Poco::toUpper(std::string{pos_before_specifier, pos}); /// should be followed by an expression ws.ignore(pos, end, expected); diff --git a/dbms/include/DB/Storages/AlterCommands.h b/dbms/include/DB/Storages/AlterCommands.h index d075135e830..e122a9f801d 100644 --- a/dbms/include/DB/Storages/AlterCommands.h +++ b/dbms/include/DB/Storages/AlterCommands.h @@ -38,7 +38,7 @@ struct AlterCommand { if (type == ADD) { - if (std::count_if(columns.begin(), columns.end(), std::bind(namesEqual, column_name, std::placeholders::_1))) + if (std::count_if(columns.begin(), columns.end(), std::bind(namesEqual, std::cref(column_name), std::placeholders::_1))) throw Exception("Cannot add column " + column_name + ": column with this name already exisits.", DB::ErrorCodes::ILLEGAL_COLUMN); @@ -54,7 +54,7 @@ struct AlterCommand /// Например "fruits.bananas" /// одинаковыми считаются имена, если они совпадают целиком или name_without_dot совпадает с частью имени до точки NamesAndTypesList::reverse_iterator reverse_insert_it = std::find_if(columns.rbegin(), columns.rend(), - std::bind(namesEqual, after_column, std::placeholders::_1)); + std::bind(namesEqual, std::cref(after_column), std::placeholders::_1)); if (reverse_insert_it == columns.rend()) throw Exception("Wrong column name. Cannot find column " + column_name + " to insert after", @@ -77,7 +77,8 @@ struct AlterCommand NamesAndTypesList::iterator column_it; do { - column_it = std::find_if(columns.begin(), columns.end(), std::bind(namesEqual, column_name, std::placeholders::_1)); + column_it = std::find_if(columns.begin(), columns.end(), + std::bind(namesEqual, std::cref(column_name), std::placeholders::_1)); if (column_it == columns.end()) { @@ -94,7 +95,7 @@ struct AlterCommand else if (type == MODIFY) { NamesAndTypesList::iterator column_it = std::find_if(columns.begin(), columns.end(), - std::bind(namesEqual, column_name, std::placeholders::_1) ); + std::bind(namesEqual, std::cref(column_name), std::placeholders::_1) ); if (column_it == columns.end()) throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.", DB::ErrorCodes::ILLEGAL_COLUMN); diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 4135b01bb6e..59890e8d469 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -30,15 +31,16 @@ InterpreterAlterQuery::InterpreterAlterQuery(ASTPtr query_ptr_, Context & contex void InterpreterAlterQuery::execute() { - ASTAlterQuery & alter = typeid_cast(*query_ptr); - String & table_name = alter.table; + auto & alter = typeid_cast(*query_ptr); + const String & table_name = alter.table; String database_name = alter.database.empty() ? context.getCurrentDatabase() : alter.database; + StoragePtr table = context.getTable(database_name, table_name); + validateColumnChanges(alter.parameters, table, context); + AlterCommands alter_commands; PartitionCommands partition_commands; parseAlter(alter.parameters, context.getDataTypeFactory(), alter_commands, partition_commands); - StoragePtr table = context.getTable(database_name, table_name); - for (const PartitionCommand & command : partition_commands) { if (command.type == PartitionCommand::DROP_PARTITION) @@ -64,11 +66,11 @@ void InterpreterAlterQuery::parseAlter( AlterCommand command; command.type = AlterCommand::ADD; - const ASTNameTypePair & ast_name_type = typeid_cast(*params.name_type); - StringRange type_range = ast_name_type.type->range; + const auto & ast_col_decl = typeid_cast(*params.col_decl); + StringRange type_range = ast_col_decl.type->range; String type_string = String(type_range.first, type_range.second - type_range.first); - command.column_name = ast_name_type.name; + command.column_name = ast_col_decl.name; command.data_type = data_type_factory.get(type_string); if (params.column) @@ -89,11 +91,11 @@ void InterpreterAlterQuery::parseAlter( AlterCommand command; command.type = AlterCommand::MODIFY; - const ASTNameTypePair & ast_name_type = typeid_cast(*params.name_type); - StringRange type_range = ast_name_type.type->range; + const auto & ast_col_decl = typeid_cast(*params.col_decl); + StringRange type_range = ast_col_decl.type->range; String type_string = String(type_range.first, type_range.second - type_range.first); - command.column_name = ast_name_type.name; + command.column_name = ast_col_decl.name; command.data_type = data_type_factory.get(type_string); out_alter_commands.push_back(command); @@ -113,6 +115,141 @@ void InterpreterAlterQuery::parseAlter( } } +void InterpreterAlterQuery::validateColumnChanges(ASTAlterQuery::ParameterContainer & params_container, const StoragePtr & table, const Context & context) +{ + auto columns = table->getColumnsList(); + columns.insert(std::end(columns), std::begin(table->materialized_columns), std::end(table->materialized_columns)); + columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns)); + auto defaults = table->column_defaults; + + std::vector> defaulted_columns{}; + + ASTPtr default_expr_list{new ASTExpressionList}; + default_expr_list->children.reserve(table->column_defaults.size()); + + for (auto & params : params_container) + { + if (params.type == ASTAlterQuery::ADD_COLUMN || params.type == ASTAlterQuery::MODIFY_COLUMN) + { + auto & col_decl = typeid_cast(*params.col_decl); + const auto & column_name = col_decl.name; + + if (params.type == ASTAlterQuery::MODIFY_COLUMN) + { + const auto it = std::find_if(std::begin(columns), std::end(columns), + std::bind(AlterCommand::namesEqual, std::cref(column_name), std::placeholders::_1)); + + if (it == std::end(columns)) + throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.", + DB::ErrorCodes::ILLEGAL_COLUMN); + + columns.erase(it); + defaults.erase(column_name); + } + + if (col_decl.type) + { + const StringRange & type_range = col_decl.type->range; + columns.emplace_back(col_decl.name, + context.getDataTypeFactory().get({type_range.first, type_range.second})); + } + + if (col_decl.default_expression) + { + + if (col_decl.type) + { + const auto tmp_column_name = col_decl.name + "_tmp"; + const auto & final_column_name = col_decl.name; + const auto conversion_function_name = "to" + columns.back().type->getName(); + + default_expr_list->children.emplace_back(setAlias( + makeASTFunction(conversion_function_name, ASTPtr{new ASTIdentifier{{}, tmp_column_name}}), + final_column_name)); + + default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), tmp_column_name)); + + defaulted_columns.emplace_back(columns.back().type.get(), &col_decl); + } + else + { + default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), col_decl.name)); + + defaulted_columns.emplace_back(nullptr, &col_decl); + } + } + } + else if (params.type == ASTAlterQuery::DROP_COLUMN) + { + const auto & column_name = typeid_cast(*(params.column)).name; + + auto found = false; + for (auto it = std::begin(columns); it != std::end(columns);) + if (AlterCommand::namesEqual(column_name, *it)) + { + found = true; + it = columns.erase(it); + } + else + ++it; + + for (auto it = std::begin(defaults); it != std::end(defaults);) + if (AlterCommand::namesEqual(column_name, { it->first, nullptr })) + it = defaults.erase(it); + else + ++it; + + if (!found) + throw Exception("Wrong column name. Cannot find column " + column_name + " to drop.", + DB::ErrorCodes::ILLEGAL_COLUMN); + } + } + + for (const auto & col_def : defaults) + default_expr_list->children.emplace_back(setAlias(col_def.second.expression->clone(), col_def.first)); + + const auto actions = ExpressionAnalyzer{default_expr_list, context, columns}.getActions(true); + const auto block = actions->getSampleBlock(); + + for (auto & column : defaulted_columns) + { + const auto type_ptr = column.first; + const auto col_decl_ptr = column.second; + + if (type_ptr) + { + const auto & tmp_column = block.getByName(col_decl_ptr->name + "_tmp"); + + /// type mismatch between explicitly specified and deduced type, add conversion + if (typeid(*type_ptr) != typeid(*tmp_column.type)) + { + col_decl_ptr->default_expression = makeASTFunction( + "to" + type_ptr->getName(), + col_decl_ptr->default_expression); + + col_decl_ptr->children.clear(); + col_decl_ptr->children.push_back(col_decl_ptr->type); + col_decl_ptr->children.push_back(col_decl_ptr->default_expression); + } + } + else + { + col_decl_ptr->type = new ASTIdentifier{}; + col_decl_ptr->query_string = new String{block.getByName(col_decl_ptr->name).type->getName()}; + col_decl_ptr->range = { + col_decl_ptr->query_string->data(), + col_decl_ptr->query_string->data() + col_decl_ptr->query_string->size() + }; + static_cast(*col_decl_ptr->type).name = *col_decl_ptr->query_string; + } + + defaults.emplace(col_decl_ptr->name, ColumnDefault{ + columnDefaultTypeFromString(col_decl_ptr->default_specifier), + setAlias(col_decl_ptr->default_expression, col_decl_ptr->name) + }); + } +} + void InterpreterAlterQuery::updateMetadata( const String & database_name, const String & table_name, const NamesAndTypesList & columns, Context & context) { diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 2577ebdc51a..6f913c875ea 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -296,10 +296,6 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( else { default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), col_decl.name)); - defaults.emplace(col_decl.name, ColumnDefault{ - columnDefaultTypeFromString(col_decl.default_specifier), - setAlias(col_decl.default_expression, col_decl.name) - }); } } } diff --git a/dbms/src/Parsers/ParserAlterQuery.cpp b/dbms/src/Parsers/ParserAlterQuery.cpp index b07554add15..ac34da7d79f 100644 --- a/dbms/src/Parsers/ParserAlterQuery.cpp +++ b/dbms/src/Parsers/ParserAlterQuery.cpp @@ -32,7 +32,7 @@ bool ParserAlterQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, Expected & e ParserIdentifier table_parser; ParserCompoundIdentifier parser_name; - ParserCompoundNameTypePair parser_name_type; + ParserCompoundColumnDeclaration parser_col_decl; ParserLiteral parser_literal; ASTPtr table; @@ -85,7 +85,7 @@ bool ParserAlterQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, Expected & e return false; ws.ignore(pos, end); - parser_name_type.parse(pos, end, params.name_type, expected); + parser_col_decl.parse(pos, end, params.col_decl, expected); ws.ignore(pos, end); if (s_after.ignore(pos, end, expected)) @@ -168,7 +168,7 @@ bool ParserAlterQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, Expected & e return false; ws.ignore(pos, end); - if (!parser_name_type.parse(pos, end, params.name_type, expected)) + if (!parser_col_decl.parse(pos, end, params.col_decl, expected)) return false; ws.ignore(pos, end); diff --git a/dbms/src/Parsers/formatAST.cpp b/dbms/src/Parsers/formatAST.cpp index f1692c25dab..0dbab1ddbc7 100644 --- a/dbms/src/Parsers/formatAST.cpp +++ b/dbms/src/Parsers/formatAST.cpp @@ -750,7 +750,7 @@ void formatAST(const ASTAlterQuery & ast, std::ostream & s, size_t indent, bo if (p.type == ASTAlterQuery::ADD_COLUMN) { s << (hilite ? hilite_keyword : "") << indent_str << "ADD COLUMN " << (hilite ? hilite_none : ""); - formatAST(*p.name_type, s, indent, hilite, true); + formatAST(*p.col_decl, s, indent, hilite, true); /// AFTER if (p.column) @@ -767,7 +767,7 @@ void formatAST(const ASTAlterQuery & ast, std::ostream & s, size_t indent, bo else if (p.type == ASTAlterQuery::MODIFY_COLUMN) { s << (hilite ? hilite_keyword : "") << indent_str << "MODIFY COLUMN " << (hilite ? hilite_none : ""); - formatAST(*p.name_type, s, indent, hilite, true); + formatAST(*p.col_decl, s, indent, hilite, true); } else if (p.type == ASTAlterQuery::DROP_PARTITION) { From 6f7997ce112a885da37d08c79f39e1100af5e792 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Fri, 10 Oct 2014 19:45:43 +0400 Subject: [PATCH 13/29] dbms: prohibit inserting materialized columns, support create as and similar queries. [#METR-12739] --- .../ProhibitColumnsBlockOutputStream.h | 39 +++++++++++++++++ .../Interpreters/InterpreterDescribeQuery.h | 1 - dbms/include/DB/Storages/ITableDeclaration.h | 14 +++--- .../DB/Storages/MergeTree/MergeTreeData.h | 2 +- dbms/include/DB/Storages/StorageChunkMerger.h | 2 +- dbms/include/DB/Storages/StorageChunkRef.h | 2 +- dbms/include/DB/Storages/StorageDistributed.h | 2 +- dbms/include/DB/Storages/StorageLog.h | 2 +- dbms/include/DB/Storages/StorageMemory.h | 2 +- dbms/include/DB/Storages/StorageMerge.h | 2 +- dbms/include/DB/Storages/StorageMergeTree.h | 2 +- dbms/include/DB/Storages/StorageNull.h | 2 +- .../DB/Storages/StorageReplicatedMergeTree.h | 2 +- .../DB/Storages/StorageSystemDatabases.h | 6 +-- .../include/DB/Storages/StorageSystemEvents.h | 6 +-- .../include/DB/Storages/StorageSystemMerges.h | 2 +- .../DB/Storages/StorageSystemNumbers.h | 6 +-- dbms/include/DB/Storages/StorageSystemOne.h | 6 +-- dbms/include/DB/Storages/StorageSystemParts.h | 2 +- .../DB/Storages/StorageSystemProcesses.h | 6 +-- .../include/DB/Storages/StorageSystemTables.h | 6 +-- dbms/include/DB/Storages/StorageTinyLog.h | 2 +- dbms/include/DB/Storages/StorageView.h | 2 +- dbms/src/Core/Block.cpp | 6 +-- .../Interpreters/InterpreterCreateQuery.cpp | 7 ++- .../Interpreters/InterpreterInsertQuery.cpp | 37 +++++++++++----- .../Interpreters/InterpreterSelectQuery.cpp | 2 +- dbms/src/Storages/ITableDeclaration.cpp | 43 ++++++++----------- dbms/src/Storages/StorageChunkMerger.cpp | 2 +- dbms/src/Storages/StorageLog.cpp | 8 ++-- dbms/src/Storages/StorageTinyLog.cpp | 8 ++-- 31 files changed, 144 insertions(+), 87 deletions(-) create mode 100644 dbms/include/DB/DataStreams/ProhibitColumnsBlockOutputStream.h diff --git a/dbms/include/DB/DataStreams/ProhibitColumnsBlockOutputStream.h b/dbms/include/DB/DataStreams/ProhibitColumnsBlockOutputStream.h new file mode 100644 index 00000000000..9a228583e5a --- /dev/null +++ b/dbms/include/DB/DataStreams/ProhibitColumnsBlockOutputStream.h @@ -0,0 +1,39 @@ +#pragma once + +#include + + +namespace DB +{ + + +/// Throws exception on encountering prohibited column in block +class ProhibitColumnsBlockOutputStream : public IBlockOutputStream +{ +public: + ProhibitColumnsBlockOutputStream(BlockOutputStreamPtr output, const NamesAndTypesList & columns) + : output{output}, columns{columns} + { + } + + void write(const Block & block) override + { + for (const auto & column : columns) + if (block.has(column.name)) + throw Exception{"Cannot insert column " + column.name, ErrorCodes::ILLEGAL_COLUMN}; + + output->write(block); + } + + void flush() { output->flush(); } + + void writePrefix() override { output->writePrefix(); } + void writeSuffix() override { output->writeSuffix(); } + +private: + BlockOutputStreamPtr output; + NamesAndTypesList columns; +}; + + +} diff --git a/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h b/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h index b9a1574813f..2f94fe854e6 100644 --- a/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h @@ -87,7 +87,6 @@ private: StoragePtr table = context.getTable(ast.database, ast.table); auto table_lock = table->lockStructure(false); columns = table->getColumnsList(); - columns.insert(std::end(columns), std::begin(table->materialized_columns), std::end(table->materialized_columns)); columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns)); column_defaults = table->column_defaults; } diff --git a/dbms/include/DB/Storages/ITableDeclaration.h b/dbms/include/DB/Storages/ITableDeclaration.h index f796040c4e9..6df2953af91 100644 --- a/dbms/include/DB/Storages/ITableDeclaration.h +++ b/dbms/include/DB/Storages/ITableDeclaration.h @@ -23,7 +23,8 @@ public: /** Получить список имён и типов столбцов таблицы, только невиртуальные. */ - virtual const NamesAndTypesList & getColumnsList() const = 0; + NamesAndTypesList getColumnsList() const; + const NamesAndTypesList & getColumnsListAsterisk() const { return getColumnsListImpl(); } /** Получить список имён столбцов таблицы, только невиртуальные. */ @@ -57,21 +58,21 @@ public: /** Проверить, что все запрошенные имена есть в таблице и заданы корректно. * (список имён не пустой и имена не повторяются) */ - void check(const Names & column_names, bool all_columns = false) const; + void check(const Names & column_names) const; /** Проверить, что все запрошенные имена есть в таблице и имеют правильные типы. */ - void check(const NamesAndTypesList & columns, bool all_columns = false) const; + void check(const NamesAndTypesList & columns) const; /** Проверить, что все имена из пересечения names и columns есть в таблице и имеют одинаковые типы. */ - void check(const NamesAndTypesList & columns, const Names & column_names, bool all_columns = false) const; + void check(const NamesAndTypesList & columns, const Names & column_names) const; /** Проверить, что блок с данными для записи содержит все столбцы таблицы с правильными типами, * содержит только столбцы таблицы, и все столбцы различны. * Если need_all, еще проверяет, что все столбцы таблицы есть в блоке. */ - void check(const Block & block, bool need_all = false, bool all_columns = false) const; + void check(const Block & block, bool need_all = false) const; virtual ~ITableDeclaration() = default; @@ -89,6 +90,9 @@ public: NamesAndTypesList materialized_columns{}; NamesAndTypesList alias_columns{}; ColumnDefaults column_defaults{}; + +private: + virtual const NamesAndTypesList & getColumnsListImpl() const = 0; }; } diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h index 9ec6fbb93e0..2ded8f80676 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h @@ -613,7 +613,7 @@ public: throw Exception("Logical error: calling method getTableName of not a table.", ErrorCodes::LOGICAL_ERROR); } - const NamesAndTypesList & getColumnsList() const { return *columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return *columns; } NameAndTypePair getColumn(const String &column_name) const { diff --git a/dbms/include/DB/Storages/StorageChunkMerger.h b/dbms/include/DB/Storages/StorageChunkMerger.h index 319fbec873c..37e1cdfd3c1 100644 --- a/dbms/include/DB/Storages/StorageChunkMerger.h +++ b/dbms/include/DB/Storages/StorageChunkMerger.h @@ -33,7 +33,7 @@ public: std::string getName() const { return "ChunkMerger"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return *columns; } + const NamesAndTypesList & getColumnsListImpl() const { return *columns; } NameAndTypePair getColumn(const String & column_name) const; bool hasColumn(const String & column_name) const; diff --git a/dbms/include/DB/Storages/StorageChunkRef.h b/dbms/include/DB/Storages/StorageChunkRef.h index 557c5edaeed..1d8f745cefa 100644 --- a/dbms/include/DB/Storages/StorageChunkRef.h +++ b/dbms/include/DB/Storages/StorageChunkRef.h @@ -17,7 +17,7 @@ public: std::string getName() const { return "ChunkRef"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return getSource().getColumnsList(); } + const NamesAndTypesList & getColumnsListImpl() const override { return getSource().getColumnsListAsterisk(); } /// В таблице, на которую мы ссылаемся, могут быть виртуальные столбцы. NameAndTypePair getColumn(const String &column_name) const { return getSource().getColumn(column_name); }; bool hasColumn(const String &column_name) const { return getSource().hasColumn(column_name); }; diff --git a/dbms/include/DB/Storages/StorageDistributed.h b/dbms/include/DB/Storages/StorageDistributed.h index 2f3c50e2ac3..0d6016edf18 100644 --- a/dbms/include/DB/Storages/StorageDistributed.h +++ b/dbms/include/DB/Storages/StorageDistributed.h @@ -49,7 +49,7 @@ public: bool supportsFinal() const { return true; } bool supportsPrewhere() const { return true; } - const NamesAndTypesList & getColumnsList() const { return *columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return *columns; } NameAndTypePair getColumn(const String &column_name) const; bool hasColumn(const String &column_name) const; diff --git a/dbms/include/DB/Storages/StorageLog.h b/dbms/include/DB/Storages/StorageLog.h index 839863a54ac..824b3adff17 100644 --- a/dbms/include/DB/Storages/StorageLog.h +++ b/dbms/include/DB/Storages/StorageLog.h @@ -151,7 +151,7 @@ public: std::string getName() const { return "Log"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return *columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return *columns; } virtual BlockInputStreams read( const Names & column_names, diff --git a/dbms/include/DB/Storages/StorageMemory.h b/dbms/include/DB/Storages/StorageMemory.h index af94a302237..78ab8264627 100644 --- a/dbms/include/DB/Storages/StorageMemory.h +++ b/dbms/include/DB/Storages/StorageMemory.h @@ -76,7 +76,7 @@ public: std::string getName() const { return "Memory"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return *columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return *columns; } size_t getSize() const { return data.size(); } diff --git a/dbms/include/DB/Storages/StorageMerge.h b/dbms/include/DB/Storages/StorageMerge.h index 049f6993bec..5b0718e8ed5 100644 --- a/dbms/include/DB/Storages/StorageMerge.h +++ b/dbms/include/DB/Storages/StorageMerge.h @@ -42,7 +42,7 @@ public: /// Проверка откладывается до метода read. Там проверяется поддержка PREWHERE у использующихся таблиц. bool supportsPrewhere() const { return true; } - const NamesAndTypesList & getColumnsList() const { return *columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return *columns; } NameAndTypePair getColumn(const String &column_name) const; bool hasColumn(const String &column_name) const; diff --git a/dbms/include/DB/Storages/StorageMergeTree.h b/dbms/include/DB/Storages/StorageMergeTree.h index 77812c074dc..7c4c7c4554b 100644 --- a/dbms/include/DB/Storages/StorageMergeTree.h +++ b/dbms/include/DB/Storages/StorageMergeTree.h @@ -54,7 +54,7 @@ public: bool supportsFinal() const { return data.supportsFinal(); } bool supportsPrewhere() const { return data.supportsPrewhere(); } - const NamesAndTypesList & getColumnsList() const { return data.getColumnsList(); } + const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListAsterisk(); } NameAndTypePair getColumn(const String & column_name) const { diff --git a/dbms/include/DB/Storages/StorageNull.h b/dbms/include/DB/Storages/StorageNull.h index 4c143ac4e75..dc5161459ed 100644 --- a/dbms/include/DB/Storages/StorageNull.h +++ b/dbms/include/DB/Storages/StorageNull.h @@ -28,7 +28,7 @@ public: std::string getName() const { return "Null"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return *columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return *columns; } BlockInputStreams read( const Names & column_names, diff --git a/dbms/include/DB/Storages/StorageReplicatedMergeTree.h b/dbms/include/DB/Storages/StorageReplicatedMergeTree.h index 9ff5f754d6b..33a670b35b0 100644 --- a/dbms/include/DB/Storages/StorageReplicatedMergeTree.h +++ b/dbms/include/DB/Storages/StorageReplicatedMergeTree.h @@ -53,7 +53,7 @@ public: bool supportsFinal() const override { return data.supportsFinal(); } bool supportsPrewhere() const override { return data.supportsPrewhere(); } - const NamesAndTypesList & getColumnsList() const override { return data.getColumnsList(); } + const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListAsterisk(); } NameAndTypePair getColumn(const String & column_name) const { diff --git a/dbms/include/DB/Storages/StorageSystemDatabases.h b/dbms/include/DB/Storages/StorageSystemDatabases.h index 50d77b37fca..314a8c11243 100644 --- a/dbms/include/DB/Storages/StorageSystemDatabases.h +++ b/dbms/include/DB/Storages/StorageSystemDatabases.h @@ -19,11 +19,11 @@ class StorageSystemDatabases : public IStorage { public: static StoragePtr create(const std::string & name_, const Context & context_); - + std::string getName() const { return "SystemDatabases"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return columns; } BlockInputStreams read( const Names & column_names, @@ -37,7 +37,7 @@ private: const std::string name; const Context & context; NamesAndTypesList columns; - + StorageSystemDatabases(const std::string & name_, const Context & context_); }; diff --git a/dbms/include/DB/Storages/StorageSystemEvents.h b/dbms/include/DB/Storages/StorageSystemEvents.h index 86f632fd04d..2c463e6ecd2 100644 --- a/dbms/include/DB/Storages/StorageSystemEvents.h +++ b/dbms/include/DB/Storages/StorageSystemEvents.h @@ -18,11 +18,11 @@ class StorageSystemEvents : public IStorage { public: static StoragePtr create(const std::string & name_); - + std::string getName() const { return "SystemEvents"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return columns; } BlockInputStreams read( const Names & column_names, @@ -35,7 +35,7 @@ public: private: const std::string name; NamesAndTypesList columns; - + StorageSystemEvents(const std::string & name_); }; diff --git a/dbms/include/DB/Storages/StorageSystemMerges.h b/dbms/include/DB/Storages/StorageSystemMerges.h index a49ee3fe6d7..b2dc8e6f82a 100644 --- a/dbms/include/DB/Storages/StorageSystemMerges.h +++ b/dbms/include/DB/Storages/StorageSystemMerges.h @@ -13,7 +13,7 @@ public: std::string getName() const override { return "SystemMerges"; } std::string getTableName() const override { return name; } - const NamesAndTypesList & getColumnsList() const override { return columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return columns; } BlockInputStreams read( const Names & column_names, ASTPtr query, diff --git a/dbms/include/DB/Storages/StorageSystemNumbers.h b/dbms/include/DB/Storages/StorageSystemNumbers.h index b6eb8d9d665..46e99dd39a0 100644 --- a/dbms/include/DB/Storages/StorageSystemNumbers.h +++ b/dbms/include/DB/Storages/StorageSystemNumbers.h @@ -19,11 +19,11 @@ class StorageSystemNumbers : public IStorage { public: static StoragePtr create(const std::string & name_, bool multithreaded_ = false); - + std::string getName() const { return "SystemNumbers"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return columns; } BlockInputStreams read( const Names & column_names, @@ -37,7 +37,7 @@ private: const std::string name; NamesAndTypesList columns; bool multithreaded; - + StorageSystemNumbers(const std::string & name_, bool multithreaded_); }; diff --git a/dbms/include/DB/Storages/StorageSystemOne.h b/dbms/include/DB/Storages/StorageSystemOne.h index ac47b081da7..8a5644a9c44 100644 --- a/dbms/include/DB/Storages/StorageSystemOne.h +++ b/dbms/include/DB/Storages/StorageSystemOne.h @@ -16,11 +16,11 @@ class StorageSystemOne : public IStorage { public: static StoragePtr create(const std::string & name_); - + std::string getName() const { return "SystemOne"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return columns; } BlockInputStreams read( const Names & column_names, @@ -33,7 +33,7 @@ public: private: const std::string name; NamesAndTypesList columns; - + StorageSystemOne(const std::string & name_); }; diff --git a/dbms/include/DB/Storages/StorageSystemParts.h b/dbms/include/DB/Storages/StorageSystemParts.h index 3b14b8bfc03..326dc86b231 100644 --- a/dbms/include/DB/Storages/StorageSystemParts.h +++ b/dbms/include/DB/Storages/StorageSystemParts.h @@ -22,7 +22,7 @@ public: std::string getName() const { return "SystemParts"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return columns; } BlockInputStreams read( const Names & column_names, diff --git a/dbms/include/DB/Storages/StorageSystemProcesses.h b/dbms/include/DB/Storages/StorageSystemProcesses.h index 3bd4c7023c0..be971e02c97 100644 --- a/dbms/include/DB/Storages/StorageSystemProcesses.h +++ b/dbms/include/DB/Storages/StorageSystemProcesses.h @@ -19,11 +19,11 @@ class StorageSystemProcesses : public IStorage { public: static StoragePtr create(const std::string & name_, const Context & context_); - + std::string getName() const { return "SystemProcesses"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return columns; } BlockInputStreams read( const Names & column_names, @@ -37,7 +37,7 @@ private: const std::string name; const Context & context; NamesAndTypesList columns; - + StorageSystemProcesses(const std::string & name_, const Context & context_); }; diff --git a/dbms/include/DB/Storages/StorageSystemTables.h b/dbms/include/DB/Storages/StorageSystemTables.h index 31018196897..ac8a5e3138e 100644 --- a/dbms/include/DB/Storages/StorageSystemTables.h +++ b/dbms/include/DB/Storages/StorageSystemTables.h @@ -19,11 +19,11 @@ class StorageSystemTables : public IStorage { public: static StoragePtr create(const std::string & name_, const Context & context_); - + std::string getName() const { return "SystemTables"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return columns; } BlockInputStreams read( const Names & column_names, @@ -37,7 +37,7 @@ private: const std::string name; const Context & context; NamesAndTypesList columns; - + StorageSystemTables(const std::string & name_, const Context & context_); }; diff --git a/dbms/include/DB/Storages/StorageTinyLog.h b/dbms/include/DB/Storages/StorageTinyLog.h index 84f23ed3e4d..7e129082bb4 100644 --- a/dbms/include/DB/Storages/StorageTinyLog.h +++ b/dbms/include/DB/Storages/StorageTinyLog.h @@ -124,7 +124,7 @@ public: std::string getName() const { return "TinyLog"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsList() const { return *columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return *columns; } BlockInputStreams read( const Names & column_names, diff --git a/dbms/include/DB/Storages/StorageView.h b/dbms/include/DB/Storages/StorageView.h index 1d352ddf144..2f7080147e7 100644 --- a/dbms/include/DB/Storages/StorageView.h +++ b/dbms/include/DB/Storages/StorageView.h @@ -22,7 +22,7 @@ public: virtual std::string getName() const { return "View"; } virtual std::string getTableName() const { return table_name; } - const NamesAndTypesList & getColumnsList() const { return *columns; } + const NamesAndTypesList & getColumnsListImpl() const override { return *columns; } DB::ASTPtr getInnerQuery() const { return inner_query.clone(); }; virtual BlockInputStreams read( diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index 327766b4bb6..cc0a0457b35 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -54,13 +54,13 @@ void Block::addDefaults(NamesAndTypesListPtr required_columns, if (default_expr_list->children.empty()) return; - /** ExpressionAnalyzer eliminates "unused" columns, in order to ensure theri safety - * we are going to operate on a copy instead of original block */ + /** ExpressionAnalyzer eliminates "unused" columns, in order to ensure their safety + * we are going to operate on a copy instead of the original block */ Block copy_block{*this}; /// evaluate default values for defaulted columns ExpressionAnalyzer{default_expr_list, context, *required_columns}.getActions(true)->execute(copy_block); - /// move evaluated columns to original block + /// move evaluated columns to the original block for (auto & column_name_type : copy_block.getColumns()) insert(std::move(column_name_type)); } diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 6f913c875ea..1192c4908f9 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -127,7 +127,12 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) column_defaults = std::move(columns_and_defaults.second); } else if (!create.as_table.empty()) - columns = new NamesAndTypesList(as_storage->getColumnsList()); + { + columns = new NamesAndTypesList(as_storage->getColumnsListAsterisk()); + materialized_columns = as_storage->materialized_columns; + alias_columns = as_storage->alias_columns; + column_defaults = as_storage->column_defaults; + } else if (create.select) { columns = new NamesAndTypesList; diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 2cbe1538b40..de13b2160c9 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -67,7 +68,8 @@ void InterpreterInsertQuery::execute(ReadBuffer * remaining_data_istr) auto table_lock = table->lockStructure(true); - BlockInputStreamPtr in; + /** @note looks suspicious, first we ask to create block from NamesAndTypesList (internally in ITableDeclaration), + * then we compose the same list from the resulting block */ NamesAndTypesListPtr required_columns = new NamesAndTypesList(table->getSampleBlock().getColumnsList()); /// Надо убедиться, что запрос идет в таблицу, которая поддерживает вставку. @@ -75,14 +77,19 @@ void InterpreterInsertQuery::execute(ReadBuffer * remaining_data_istr) table->write(query_ptr); /// Создаем кортеж из нескольких стримов, в которые будем писать данные. - BlockOutputStreamPtr out{new AddingDefaultBlockOutputStream{ - new PushingToViewsBlockOutputStream{query.database, query.table, context, query_ptr}, - required_columns, table->column_defaults, context - }}; + BlockOutputStreamPtr out{ + new ProhibitColumnsBlockOutputStream{ + new AddingDefaultBlockOutputStream{ + new PushingToViewsBlockOutputStream{query.database, query.table, context, query_ptr}, + required_columns, table->column_defaults, context + }, table->materialized_columns + } + }; /// Какой тип запроса: INSERT VALUES | INSERT FORMAT | INSERT SELECT? if (!query.select) { + String format = query.format; if (format.empty()) format = "Values"; @@ -103,13 +110,17 @@ void InterpreterInsertQuery::execute(ReadBuffer * remaining_data_istr) ConcatReadBuffer istr(buffers); Block sample = getSampleBlock(); - in = context.getFormatFactory().getInput(format, istr, sample, context.getSettings().max_insert_block_size, context.getDataTypeFactory()); + BlockInputStreamPtr in{ + context.getFormatFactory().getInput( + format, istr, sample, context.getSettings().max_insert_block_size, + context.getDataTypeFactory()) + }; copyData(*in, *out); } else { InterpreterSelectQuery interpreter_select(query.select, context); - in = interpreter_select.execute(); + BlockInputStreamPtr in{interpreter_select.execute()}; in = new MaterializingBlockInputStream(in); copyData(*in, *out); @@ -131,10 +142,14 @@ BlockIO InterpreterInsertQuery::execute() table->write(query_ptr); /// Создаем кортеж из нескольких стримов, в которые будем писать данные. - BlockOutputStreamPtr out{new AddingDefaultBlockOutputStream{ - new PushingToViewsBlockOutputStream{query.database, query.table, context, query_ptr}, - required_columns, table->column_defaults, context - }}; + BlockOutputStreamPtr out{ + new ProhibitColumnsBlockOutputStream{ + new AddingDefaultBlockOutputStream{ + new PushingToViewsBlockOutputStream{query.database, query.table, context, query_ptr}, + required_columns, table->column_defaults, context + }, table->materialized_columns + } + }; BlockIO res; res.out_sample = getSampleBlock(); diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 997e11c8f05..0e24558adb4 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -67,7 +67,7 @@ void InterpreterSelectQuery::init(BlockInputStreamPtr input_, const NamesAndType table_lock = storage->lockStructure(false); if (table_column_names.empty()) - context.setColumns(storage->getColumnsList()); + context.setColumns(storage->getColumnsListAsterisk()); } if (!table_column_names.empty()) diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index c9383f5e3d9..4568d4d371e 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -11,6 +11,15 @@ namespace DB { +NamesAndTypesList ITableDeclaration::getColumnsList() const +{ + auto columns = getColumnsListImpl(); + columns.insert(std::end(columns), std::begin(materialized_columns), std::end(materialized_columns)); + + return columns; +} + + bool ITableDeclaration::hasRealColumn(const String & column_name) const { const NamesAndTypesList & real_columns = getColumnsList(); @@ -60,17 +69,12 @@ bool ITableDeclaration::hasMaterializedColumn(const String & column_name) const bool ITableDeclaration::hasColumn(const String & column_name) const { - return hasRealColumn(column_name) || hasMaterializedColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. + return hasRealColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. } NameAndTypePair ITableDeclaration::getColumn(const String & column_name) const { - const auto it = column_defaults.find(column_name); - - if (it == std::end(column_defaults) || it->second.type == ColumnDefaultType::Default) - return getRealColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. - - return getMaterializedColumn(column_name); + return getRealColumn(column_name); /// По умолчанию считаем, что виртуальных столбцов в сторадже нет. } @@ -144,7 +148,7 @@ static NamesAndTypesMap getColumnsMap(const Args &... args) } -void ITableDeclaration::check(const Names & column_names, const bool all_columns) const +void ITableDeclaration::check(const Names & column_names) const { const NamesAndTypesList & available_columns = getColumnsList(); @@ -152,9 +156,7 @@ void ITableDeclaration::check(const Names & column_names, const bool all_columns throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns), ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED); - const auto columns_map = all_columns ? - getColumnsMap(available_columns, materialized_columns) : - getColumnsMap(available_columns); + const auto columns_map = getColumnsMap(available_columns); typedef google::dense_hash_set UniqueStrings; UniqueStrings unique_names; @@ -174,12 +176,10 @@ void ITableDeclaration::check(const Names & column_names, const bool all_columns } -void ITableDeclaration::check(const NamesAndTypesList & columns, const bool all_columns) const +void ITableDeclaration::check(const NamesAndTypesList & columns) const { const NamesAndTypesList & available_columns = getColumnsList(); - const auto columns_map = all_columns ? - getColumnsMap(available_columns, materialized_columns) : - getColumnsMap(available_columns); + const auto columns_map = getColumnsMap(available_columns); typedef google::dense_hash_set UniqueStrings; UniqueStrings unique_names; @@ -204,12 +204,10 @@ void ITableDeclaration::check(const NamesAndTypesList & columns, const bool all_ } -void ITableDeclaration::check(const NamesAndTypesList & columns, const Names & column_names, const bool all_columns) const +void ITableDeclaration::check(const NamesAndTypesList & columns, const Names & column_names) const { const NamesAndTypesList & available_columns = getColumnsList(); - const auto available_columns_map = all_columns ? - getColumnsMap(available_columns, materialized_columns) : - getColumnsMap(available_columns); + const auto available_columns_map = getColumnsMap(available_columns); const NamesAndTypesMap & provided_columns_map = getColumnsMap(columns); if (column_names.empty()) @@ -243,13 +241,10 @@ void ITableDeclaration::check(const NamesAndTypesList & columns, const Names & c } -void ITableDeclaration::check(const Block & block, bool need_all, const bool all_columns) const +void ITableDeclaration::check(const Block & block, bool need_all) const { const NamesAndTypesList & available_columns = getColumnsList(); - const auto columns_map = all_columns ? - getColumnsMap(available_columns, materialized_columns) : - getColumnsMap(available_columns); - + const auto columns_map = getColumnsMap(available_columns); typedef std::unordered_set NameSet; NameSet names_in_block; diff --git a/dbms/src/Storages/StorageChunkMerger.cpp b/dbms/src/Storages/StorageChunkMerger.cpp index ba4ecf96924..4a2c4e81570 100644 --- a/dbms/src/Storages/StorageChunkMerger.cpp +++ b/dbms/src/Storages/StorageChunkMerger.cpp @@ -369,7 +369,7 @@ bool StorageChunkMerger::mergeChunks(const Storages & chunks) /// Объединим множества столбцов сливаемых чанков. ColumnsMap known_columns_types; - for (const NameAndTypePair & column : *columns) + for (const NameAndTypePair & column : getColumnsList()) known_columns_types.insert(std::make_pair(column.name, column.type)); NamesAndTypesListPtr required_columns = new NamesAndTypesList; diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 26e303e72b2..049540736bb 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -250,8 +250,8 @@ LogBlockOutputStream::LogBlockOutputStream(StorageLog & storage_) : storage(storage_), lock(storage.rwlock), marks_stream(storage.marks_file.path(), 4096, O_APPEND | O_CREAT | O_WRONLY) { - for (NamesAndTypesList::const_iterator it = storage.columns->begin(); it != storage.columns->end(); ++it) - addStream(it->name, *it->type); + for (const auto & column : storage.getColumnsList()) + addStream(column.name, *column.type); } @@ -428,8 +428,8 @@ StorageLog::StorageLog( /// создаём файлы, если их нет Poco::File(path + escapeForFileName(name) + '/').createDirectories(); - for (NamesAndTypesList::const_iterator it = columns->begin(); it != columns->end(); ++it) - addFile(it->name, *it->type); + for (const auto & column : getColumnsList()) + addFile(column.name, *column.type); marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); } diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index faf19a0d695..ce8062e3a65 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -194,8 +194,8 @@ void TinyLogBlockInputStream::readData(const String & name, const IDataType & ty TinyLogBlockOutputStream::TinyLogBlockOutputStream(StorageTinyLog & storage_) : storage(storage_) { - for (NamesAndTypesList::const_iterator it = storage.columns->begin(); it != storage.columns->end(); ++it) - addStream(it->name, *it->type); + for (const auto & col : storage.getColumnsList()) + addStream(col.name, *col.type); } @@ -323,8 +323,8 @@ StorageTinyLog::StorageTinyLog( throwFromErrno("Cannot create directory " + full_path, ErrorCodes::CANNOT_CREATE_DIRECTORY); } - for (NamesAndTypesList::const_iterator it = columns->begin(); it != columns->end(); ++it) - addFile(it->name, *it->type); + for (const auto & col : getColumnsList()) + addFile(col.name, *col.type); } StoragePtr StorageTinyLog::create( From 43deee45c98a54b7a4b49a861942a7dc1fde60f4 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Mon, 13 Oct 2014 18:35:40 +0400 Subject: [PATCH 14/29] dbms: MATERIALIZED columns evaluation during insert, error handling. #[METR-12739] --- .../MaterializingBlockOutputStream.h | 37 +++++ .../ProhibitColumnsBlockOutputStream.h | 6 +- .../DB/Functions/FunctionsMiscellaneous.h | 3 +- dbms/include/DB/Storages/ITableDeclaration.h | 1 + .../Interpreters/InterpreterInsertQuery.cpp | 27 ++-- dbms/src/Storages/ITableDeclaration.cpp | 26 ++-- .../0_stateless/00030_1_alter_table.reference | 30 ++-- .../0_stateless/00030_2_alter_table.reference | 22 +-- .../00061_merge_tree_alter.reference | 142 +++++++++--------- 9 files changed, 169 insertions(+), 125 deletions(-) create mode 100644 dbms/include/DB/DataStreams/MaterializingBlockOutputStream.h diff --git a/dbms/include/DB/DataStreams/MaterializingBlockOutputStream.h b/dbms/include/DB/DataStreams/MaterializingBlockOutputStream.h new file mode 100644 index 00000000000..641a522373a --- /dev/null +++ b/dbms/include/DB/DataStreams/MaterializingBlockOutputStream.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +/** Преобразует столбцы-константы в полноценные столбцы ("материализует" их). + */ +class MaterializingBlockOutputStream : public IBlockOutputStream +{ +public: + MaterializingBlockOutputStream(const BlockOutputStreamPtr & output) : output{output} + {} + + void write(const Block & original_block) override + { + /// copy block to get rid of const + auto block = original_block; + + for (const auto i : ext::range(0, block.columns())) + { + ColumnPtr & col = block.getByPosition(i).column; + if (col->isConst()) + col = dynamic_cast(*col).convertToFullColumn(); + } + + output->write(block); + } + + BlockOutputStreamPtr output; +}; + +} diff --git a/dbms/include/DB/DataStreams/ProhibitColumnsBlockOutputStream.h b/dbms/include/DB/DataStreams/ProhibitColumnsBlockOutputStream.h index 9a228583e5a..296aa202fe5 100644 --- a/dbms/include/DB/DataStreams/ProhibitColumnsBlockOutputStream.h +++ b/dbms/include/DB/DataStreams/ProhibitColumnsBlockOutputStream.h @@ -11,11 +11,12 @@ namespace DB class ProhibitColumnsBlockOutputStream : public IBlockOutputStream { public: - ProhibitColumnsBlockOutputStream(BlockOutputStreamPtr output, const NamesAndTypesList & columns) + ProhibitColumnsBlockOutputStream(const BlockOutputStreamPtr & output, const NamesAndTypesList & columns) : output{output}, columns{columns} { } +private: void write(const Block & block) override { for (const auto & column : columns) @@ -25,12 +26,11 @@ public: output->write(block); } - void flush() { output->flush(); } + void flush() override { output->flush(); } void writePrefix() override { output->writePrefix(); } void writeSuffix() override { output->writeSuffix(); } -private: BlockOutputStreamPtr output; NamesAndTypesList columns; }; diff --git a/dbms/include/DB/Functions/FunctionsMiscellaneous.h b/dbms/include/DB/Functions/FunctionsMiscellaneous.h index 088ce7a2127..cc367d8ed5a 100644 --- a/dbms/include/DB/Functions/FunctionsMiscellaneous.h +++ b/dbms/include/DB/Functions/FunctionsMiscellaneous.h @@ -123,7 +123,8 @@ public: return new DataTypeString; } - /// Выполнить функцию над блоком. + /** Выполнить функцию над блоком. convertToFullColumn вызывается для того, чтобы в случае + * распределенного выполнения запроса каждый сервер возвращал свое имя хоста. */ void execute(Block & block, const ColumnNumbers & arguments, size_t result) { block.getByPosition(result).column = ColumnConstString( diff --git a/dbms/include/DB/Storages/ITableDeclaration.h b/dbms/include/DB/Storages/ITableDeclaration.h index 6df2953af91..5379e67ccb9 100644 --- a/dbms/include/DB/Storages/ITableDeclaration.h +++ b/dbms/include/DB/Storages/ITableDeclaration.h @@ -54,6 +54,7 @@ public: /** То же самое, но в виде блока-образца. */ Block getSampleBlock() const; + Block getSampleBlockNonMaterialized() const; /** Проверить, что все запрошенные имена есть в таблице и заданы корректно. * (список имён не пустой и имена не повторяются) diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index de13b2160c9..66bcfb2b468 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -1,8 +1,8 @@ #include #include +#include #include -#include #include #include @@ -35,11 +35,12 @@ StoragePtr InterpreterInsertQuery::getTable() Block InterpreterInsertQuery::getSampleBlock() { ASTInsertQuery & query = typeid_cast(*query_ptr); - Block db_sample = getTable()->getSampleBlock(); /// Если в запросе не указана информация о столбцах if (!query.columns) - return db_sample; + return getTable()->getSampleBlockNonMaterialized(); + + Block db_sample = getTable()->getSampleBlock(); /// Формируем блок, основываясь на именах столбцов из запроса Block res; @@ -70,7 +71,7 @@ void InterpreterInsertQuery::execute(ReadBuffer * remaining_data_istr) /** @note looks suspicious, first we ask to create block from NamesAndTypesList (internally in ITableDeclaration), * then we compose the same list from the resulting block */ - NamesAndTypesListPtr required_columns = new NamesAndTypesList(table->getSampleBlock().getColumnsList()); + NamesAndTypesListPtr required_columns = new NamesAndTypesList(table->getColumnsList()); /// Надо убедиться, что запрос идет в таблицу, которая поддерживает вставку. /// TODO Плохо - исправить. @@ -80,9 +81,12 @@ void InterpreterInsertQuery::execute(ReadBuffer * remaining_data_istr) BlockOutputStreamPtr out{ new ProhibitColumnsBlockOutputStream{ new AddingDefaultBlockOutputStream{ - new PushingToViewsBlockOutputStream{query.database, query.table, context, query_ptr}, + new MaterializingBlockOutputStream{ + new PushingToViewsBlockOutputStream{query.database, query.table, context, query_ptr} + }, required_columns, table->column_defaults, context - }, table->materialized_columns + }, + table->materialized_columns } }; @@ -121,7 +125,6 @@ void InterpreterInsertQuery::execute(ReadBuffer * remaining_data_istr) { InterpreterSelectQuery interpreter_select(query.select, context); BlockInputStreamPtr in{interpreter_select.execute()}; - in = new MaterializingBlockInputStream(in); copyData(*in, *out); } @@ -135,7 +138,7 @@ BlockIO InterpreterInsertQuery::execute() auto table_lock = table->lockStructure(true); - NamesAndTypesListPtr required_columns = new NamesAndTypesList(table->getSampleBlock().getColumnsList()); + NamesAndTypesListPtr required_columns = new NamesAndTypesList(table->getColumnsList()); /// Надо убедиться, что запрос идет в таблицу, которая поддерживает вставку. /// TODO Плохо - исправить. @@ -145,9 +148,12 @@ BlockIO InterpreterInsertQuery::execute() BlockOutputStreamPtr out{ new ProhibitColumnsBlockOutputStream{ new AddingDefaultBlockOutputStream{ - new PushingToViewsBlockOutputStream{query.database, query.table, context, query_ptr}, + new MaterializingBlockOutputStream{ + new PushingToViewsBlockOutputStream{query.database, query.table, context, query_ptr} + }, required_columns, table->column_defaults, context - }, table->materialized_columns + }, + table->materialized_columns } }; @@ -163,7 +169,6 @@ BlockIO InterpreterInsertQuery::execute() { InterpreterSelectQuery interpreter_select(query.select, context); BlockInputStreamPtr in = interpreter_select.execute(); - in = new MaterializingBlockInputStream(in); copyData(*in, *out); } diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index 4568d4d371e..0a6144b20b8 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -84,10 +84,6 @@ const DataTypePtr ITableDeclaration::getDataTypeByName(const String & column_nam if (column.name == column_name) return column.type; - for (const auto & column : materialized_columns) - if (column.name == column_name) - return column.type; - throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); } @@ -95,16 +91,20 @@ const DataTypePtr ITableDeclaration::getDataTypeByName(const String & column_nam Block ITableDeclaration::getSampleBlock() const { Block res; - const NamesAndTypesList & names_and_types = getColumnsList(); - for (NamesAndTypesList::const_iterator it = names_and_types.begin(); it != names_and_types.end(); ++it) - { - ColumnWithNameAndType col; - col.name = it->name; - col.type = it->type; - col.column = col.type->createColumn(); - res.insert(col); - } + for (const auto & col : getColumnsList()) + res.insert({ col.type->createColumn(), col.type, col.name }); + + return res; +} + + +Block ITableDeclaration::getSampleBlockNonMaterialized() const +{ + Block res; + + for (const auto & col : getColumnsListAsterisk()) + res.insert({ col.type->createColumn(), col.type, col.name }); return res; } diff --git a/dbms/tests/queries/0_stateless/00030_1_alter_table.reference b/dbms/tests/queries/0_stateless/00030_1_alter_table.reference index cbbddce5939..46a966e23c6 100644 --- a/dbms/tests/queries/0_stateless/00030_1_alter_table.reference +++ b/dbms/tests/queries/0_stateless/00030_1_alter_table.reference @@ -1,15 +1,15 @@ -CounterID UInt32 -StartDate Date -UserID UInt32 -VisitID UInt32 -NestedColumn.A Array(UInt8) -NestedColumn.S Array(String) -ToDrop UInt32 -Added0 UInt32 -Added1 UInt32 -Added2 UInt32 -AddedNested1.A Array(UInt32) -AddedNested1.B Array(UInt64) -AddedNested1.C Array(String) -AddedNested2.A Array(UInt32) -AddedNested2.B Array(UInt64) +CounterID UInt32 +StartDate Date +UserID UInt32 +VisitID UInt32 +NestedColumn.A Array(UInt8) +NestedColumn.S Array(String) +ToDrop UInt32 +Added0 UInt32 +Added1 UInt32 +Added2 UInt32 +AddedNested1.A Array(UInt32) +AddedNested1.B Array(UInt64) +AddedNested1.C Array(String) +AddedNested2.A Array(UInt32) +AddedNested2.B Array(UInt64) diff --git a/dbms/tests/queries/0_stateless/00030_2_alter_table.reference b/dbms/tests/queries/0_stateless/00030_2_alter_table.reference index fad8ec5ca08..545979609b8 100644 --- a/dbms/tests/queries/0_stateless/00030_2_alter_table.reference +++ b/dbms/tests/queries/0_stateless/00030_2_alter_table.reference @@ -1,11 +1,11 @@ -CounterID UInt32 -StartDate Date -UserID UInt32 -VisitID UInt32 -Added0 String -Added1 UInt32 -Added2 UInt32 -AddedNested1.A Array(UInt32) -AddedNested1.C Array(String) -AddedNested2.A Array(UInt32) -AddedNested2.B Array(UInt64) +CounterID UInt32 +StartDate Date +UserID UInt32 +VisitID UInt32 +Added0 String +Added1 UInt32 +Added2 UInt32 +AddedNested1.A Array(UInt32) +AddedNested1.C Array(String) +AddedNested2.A Array(UInt32) +AddedNested2.B Array(UInt64) diff --git a/dbms/tests/queries/0_stateless/00061_merge_tree_alter.reference b/dbms/tests/queries/0_stateless/00061_merge_tree_alter.reference index 09a283fd959..aecf2f3c4e8 100644 --- a/dbms/tests/queries/0_stateless/00061_merge_tree_alter.reference +++ b/dbms/tests/queries/0_stateless/00061_merge_tree_alter.reference @@ -1,72 +1,72 @@ -d Date -k UInt64 -i32 Int32 +d Date +k UInt64 +i32 Int32 CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32) ENGINE = MergeTree(d, k, 8192) 2015-01-01 10 42 -d Date -k UInt64 -i32 Int32 -dt DateTime +d Date +k UInt64 +i32 Int32 +dt DateTime CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime) ENGINE = MergeTree(d, k, 8192) 2015-01-01 9 41 1992-01-01 08:00:00 2015-01-01 10 42 0000-00-00 00:00:00 -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String)) ENGINE = MergeTree(d, k, 8192) 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] 2015-01-01 10 42 0000-00-00 00:00:00 [] [] -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -n.d Array(Date) +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +n.d Array(Date) CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date)) ENGINE = MergeTree(d, k, 8192) 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] [] 2015-01-01 10 42 0000-00-00 00:00:00 [] [] [] -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -n.d Array(Date) -s String +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +n.d Array(Date) +s String CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date), s String) ENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] ['2000-01-01','2000-01-01','2000-01-03'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] [] 2015-01-01 10 42 0000-00-00 00:00:00 [] [] [] -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -s Int64 +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +s Int64 CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), s Int64) ENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] 0 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 0 2015-01-01 9 41 1992-01-01 08:00:00 [] [] 0 2015-01-01 10 42 0000-00-00 00:00:00 [] [] 0 -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -s UInt32 -n.d Array(Date) +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +s UInt32 +n.d Array(Date) CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), s UInt32, `n.d` Array(Date)) ENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] 0 ['0000-00-00','0000-00-00','0000-00-00'] @@ -78,58 +78,58 @@ CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 0 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] 0 [] 2015-01-01 10 42 0000-00-00 00:00:00 [] [] 0 [] -d Date -k UInt64 -i32 Int32 -dt DateTime -n.s Array(String) -s UInt32 +d Date +k UInt64 +i32 Int32 +dt DateTime +n.s Array(String) +s UInt32 CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime, `n.s` Array(String), s UInt32) ENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 ['asd','qwe','qwe'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 ['120','130','140'] 0 2015-01-01 8 40 2012-12-12 12:12:12 ['12','13','14'] 0 2015-01-01 9 41 1992-01-01 08:00:00 [] 0 2015-01-01 10 42 0000-00-00 00:00:00 [] 0 -d Date -k UInt64 -i32 Int32 -dt DateTime -s UInt32 +d Date +k UInt64 +i32 Int32 +dt DateTime +s UInt32 CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime, s UInt32) ENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 2015-01-01 7 39 2014-07-14 13:26:50 0 2015-01-01 8 40 2012-12-12 12:12:12 0 2015-01-01 9 41 1992-01-01 08:00:00 0 2015-01-01 10 42 0000-00-00 00:00:00 0 -d Date -k UInt64 -i32 Int32 -dt DateTime -s UInt32 -n.s Array(String) -n.d Array(Date) +d Date +k UInt64 +i32 Int32 +dt DateTime +s UInt32 +n.s Array(String) +n.d Array(Date) CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime, s UInt32, `n.s` Array(String), `n.d` Array(Date)) ENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 [] [] 2015-01-01 7 39 2014-07-14 13:26:50 0 [] [] 2015-01-01 8 40 2012-12-12 12:12:12 0 [] [] 2015-01-01 9 41 1992-01-01 08:00:00 0 [] [] 2015-01-01 10 42 0000-00-00 00:00:00 0 [] [] -d Date -k UInt64 -i32 Int32 -dt DateTime -s UInt32 +d Date +k UInt64 +i32 Int32 +dt DateTime +s UInt32 CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt DateTime, s UInt32) ENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 2015-01-01 7 39 2014-07-14 13:26:50 0 2015-01-01 8 40 2012-12-12 12:12:12 0 2015-01-01 9 41 1992-01-01 08:00:00 0 2015-01-01 10 42 0000-00-00 00:00:00 0 -d Date -k UInt64 -i32 Int32 -dt Date -s DateTime +d Date +k UInt64 +i32 Int32 +dt Date +s DateTime CREATE TABLE test.alter ( d Date, k UInt64, i32 Int32, dt Date, s DateTime) ENGINE = MergeTree(d, k, 8192) 2015-01-01 6 38 2014-07-15 1970-01-02 06:55:00 2015-01-01 7 39 2014-07-14 0000-00-00 00:00:00 From 0bfd35e5c7459352cc6e0b5627ef22ea8874e6eb Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 16 Oct 2014 17:37:01 +0400 Subject: [PATCH 15/29] dbms: defaulted columns support for ALTER, currently without type deduction and checking. --- .../DB/Interpreters/InterpreterAlterQuery.h | 11 +- dbms/include/DB/Storages/AlterCommands.h | 92 ++------- dbms/include/DB/Storages/ColumnDefault.h | 2 +- dbms/include/DB/Storages/ITableDeclaration.h | 2 +- dbms/include/DB/Storages/StorageChunkRef.h | 2 +- dbms/include/DB/Storages/StorageMergeTree.h | 2 +- .../DB/Storages/StorageReplicatedMergeTree.h | 2 +- dbms/src/Core/Block.cpp | 3 +- .../Interpreters/InterpreterAlterQuery.cpp | 175 ++++-------------- .../Interpreters/InterpreterCreateQuery.cpp | 2 +- .../Interpreters/InterpreterSelectQuery.cpp | 2 +- dbms/src/Storages/AlterCommands.cpp | 151 +++++++++++++++ dbms/src/Storages/ITableDeclaration.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 7 +- .../MergeTree/MergeTreePartChecker.cpp | 2 + dbms/src/Storages/StorageDistributed.cpp | 5 +- dbms/src/Storages/StorageMerge.cpp | 5 +- dbms/src/Storages/StorageMergeTree.cpp | 19 +- .../Storages/StorageReplicatedMergeTree.cpp | 44 ++++- dbms/src/Storages/StorageView.cpp | 2 +- 20 files changed, 278 insertions(+), 254 deletions(-) create mode 100644 dbms/src/Storages/AlterCommands.cpp diff --git a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h index 9da89aaccb7..288f577b34d 100644 --- a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h @@ -21,7 +21,13 @@ public: /** Изменяет список столбцов в метаданных таблицы на диске. Нужно вызывать под TableStructureLock соответствующей таблицы. */ - static void updateMetadata(const String & database, const String & table, const NamesAndTypesList & columns, Context & context); + static void updateMetadata(const String & database, + const String & table, + const NamesAndTypesList & columns, + const NamesAndTypesList & materialized_columns, + const NamesAndTypesList & alias_columns, + const ColumnDefaults & column_defaults, + Context & context); private: struct PartitionCommand { @@ -58,7 +64,6 @@ private: static void parseAlter(const ASTAlterQuery::ParameterContainer & params, const DataTypeFactory & data_type_factory, AlterCommands & out_alter_commands, PartitionCommands & out_partition_commands); - - static void validateColumnChanges(ASTAlterQuery::ParameterContainer & params, const StoragePtr & table, const Context & context); }; + } diff --git a/dbms/include/DB/Storages/AlterCommands.h b/dbms/include/DB/Storages/AlterCommands.h index e122a9f801d..8f67c021e53 100644 --- a/dbms/include/DB/Storages/AlterCommands.h +++ b/dbms/include/DB/Storages/AlterCommands.h @@ -1,7 +1,7 @@ #pragma once + #include -#include -#include +#include namespace DB { @@ -23,6 +23,9 @@ struct AlterCommand /// Для ADD и MODIFY - новый тип столбца. DataTypePtr data_type; + ColumnDefaultType default_type{}; + ASTPtr default_expression{}; + /// Для ADD - после какого столбца добавить новый. Если пустая строка, добавить в конец. Добавить в начало сейчас нельзя. String after_column; @@ -34,88 +37,19 @@ struct AlterCommand return (name_with_dot == name_type.name.substr(0, name_without_dot.length() + 1) || name_without_dot == name_type.name); } - void apply(NamesAndTypesList & columns) const - { - if (type == ADD) - { - if (std::count_if(columns.begin(), columns.end(), std::bind(namesEqual, std::cref(column_name), std::placeholders::_1))) - throw Exception("Cannot add column " + column_name + ": column with this name already exisits.", - DB::ErrorCodes::ILLEGAL_COLUMN); - - if (DataTypeNested::extractNestedTableName(column_name) != column_name && - !typeid_cast(&*data_type)) - throw Exception("Can't add nested column " + column_name + " of non-array type " + data_type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - NamesAndTypesList::iterator insert_it = columns.end(); - if (!after_column.empty()) - { - /// Пытаемся найти первую с конца колонку с именем column_name или с именем, начинающимся с column_name и ".". - /// Например "fruits.bananas" - /// одинаковыми считаются имена, если они совпадают целиком или name_without_dot совпадает с частью имени до точки - NamesAndTypesList::reverse_iterator reverse_insert_it = std::find_if(columns.rbegin(), columns.rend(), - std::bind(namesEqual, std::cref(after_column), std::placeholders::_1)); - - if (reverse_insert_it == columns.rend()) - throw Exception("Wrong column name. Cannot find column " + column_name + " to insert after", - DB::ErrorCodes::ILLEGAL_COLUMN); - else - { - /// base возвращает итератор, уже смещенный на один элемент вправо - insert_it = reverse_insert_it.base(); - } - } - - columns.insert(insert_it, NameAndTypePair(column_name, data_type)); - - /// Медленно, так как каждый раз копируется список - columns = *DataTypeNested::expandNestedColumns(columns); - } - else if (type == DROP) - { - bool is_first = true; - NamesAndTypesList::iterator column_it; - do - { - column_it = std::find_if(columns.begin(), columns.end(), - std::bind(namesEqual, std::cref(column_name), std::placeholders::_1)); - - if (column_it == columns.end()) - { - if (is_first) - throw Exception("Wrong column name. Cannot find column " + column_name + " to drop", - DB::ErrorCodes::ILLEGAL_COLUMN); - } - else - columns.erase(column_it); - is_first = false; - } - while (column_it != columns.end()); - } - else if (type == MODIFY) - { - NamesAndTypesList::iterator column_it = std::find_if(columns.begin(), columns.end(), - std::bind(namesEqual, std::cref(column_name), std::placeholders::_1) ); - if (column_it == columns.end()) - throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.", - DB::ErrorCodes::ILLEGAL_COLUMN); - column_it->type = data_type; - } - else - throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR); - } + void apply(NamesAndTypesList & columns, + NamesAndTypesList & materialized_columns, + NamesAndTypesList & alias_columns, + ColumnDefaults & column_defaults) const; }; class AlterCommands : public std::vector { public: - void apply(NamesAndTypesList & columns) const - { - NamesAndTypesList new_columns = columns; - for (const AlterCommand & command : *this) - command.apply(new_columns); - columns = new_columns; - } + void apply(NamesAndTypesList & columns, + NamesAndTypesList & materialized_columns, + NamesAndTypesList & alias_columns, + ColumnDefaults & column_defaults) const; }; } diff --git a/dbms/include/DB/Storages/ColumnDefault.h b/dbms/include/DB/Storages/ColumnDefault.h index c6dd3f986c7..68b667fea26 100644 --- a/dbms/include/DB/Storages/ColumnDefault.h +++ b/dbms/include/DB/Storages/ColumnDefault.h @@ -26,7 +26,7 @@ namespace std namespace DB { - inline ColumnDefaultType columnDefaultTypeFromString(const String& str) + inline ColumnDefaultType columnDefaultTypeFromString(const String & str) { static const std::unordered_map map{ { "DEFAULT", ColumnDefaultType::Default }, diff --git a/dbms/include/DB/Storages/ITableDeclaration.h b/dbms/include/DB/Storages/ITableDeclaration.h index 5379e67ccb9..3af824b1ed3 100644 --- a/dbms/include/DB/Storages/ITableDeclaration.h +++ b/dbms/include/DB/Storages/ITableDeclaration.h @@ -24,7 +24,7 @@ public: /** Получить список имён и типов столбцов таблицы, только невиртуальные. */ NamesAndTypesList getColumnsList() const; - const NamesAndTypesList & getColumnsListAsterisk() const { return getColumnsListImpl(); } + const NamesAndTypesList & getColumnsListNonMaterialized() const { return getColumnsListImpl(); } /** Получить список имён столбцов таблицы, только невиртуальные. */ diff --git a/dbms/include/DB/Storages/StorageChunkRef.h b/dbms/include/DB/Storages/StorageChunkRef.h index 1d8f745cefa..d255d50b67a 100644 --- a/dbms/include/DB/Storages/StorageChunkRef.h +++ b/dbms/include/DB/Storages/StorageChunkRef.h @@ -17,7 +17,7 @@ public: std::string getName() const { return "ChunkRef"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsListImpl() const override { return getSource().getColumnsListAsterisk(); } + const NamesAndTypesList & getColumnsListImpl() const override { return getSource().getColumnsListNonMaterialized(); } /// В таблице, на которую мы ссылаемся, могут быть виртуальные столбцы. NameAndTypePair getColumn(const String &column_name) const { return getSource().getColumn(column_name); }; bool hasColumn(const String &column_name) const { return getSource().hasColumn(column_name); }; diff --git a/dbms/include/DB/Storages/StorageMergeTree.h b/dbms/include/DB/Storages/StorageMergeTree.h index 7c4c7c4554b..17dfc316eea 100644 --- a/dbms/include/DB/Storages/StorageMergeTree.h +++ b/dbms/include/DB/Storages/StorageMergeTree.h @@ -54,7 +54,7 @@ public: bool supportsFinal() const { return data.supportsFinal(); } bool supportsPrewhere() const { return data.supportsPrewhere(); } - const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListAsterisk(); } + const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListNonMaterialized(); } NameAndTypePair getColumn(const String & column_name) const { diff --git a/dbms/include/DB/Storages/StorageReplicatedMergeTree.h b/dbms/include/DB/Storages/StorageReplicatedMergeTree.h index 33a670b35b0..a8e2856ae69 100644 --- a/dbms/include/DB/Storages/StorageReplicatedMergeTree.h +++ b/dbms/include/DB/Storages/StorageReplicatedMergeTree.h @@ -53,7 +53,7 @@ public: bool supportsFinal() const override { return data.supportsFinal(); } bool supportsPrewhere() const override { return data.supportsPrewhere(); } - const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListAsterisk(); } + const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListNonMaterialized(); } NameAndTypePair getColumn(const String & column_name) const { diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index cc0a0457b35..fdb22ac0c81 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -44,10 +44,11 @@ void Block::addDefaults(NamesAndTypesListPtr required_columns, const auto it = column_defaults.find(column.name); + /// expressions must be cloned to prevent modification by ExpressionAnalyzer if (it == column_defaults.end()) insertDefault(column.name, column.type); else - default_expr_list->children.emplace_back(it->second.expression); + default_expr_list->children.emplace_back(it->second.expression->clone()); } /// nothing to evaluate diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 59890e8d469..5ebe338d5f8 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -35,7 +35,6 @@ void InterpreterAlterQuery::execute() const String & table_name = alter.table; String database_name = alter.database.empty() ? context.getCurrentDatabase() : alter.database; StoragePtr table = context.getTable(database_name, table_name); - validateColumnChanges(alter.parameters, table, context); AlterCommands alter_commands; PartitionCommands partition_commands; @@ -67,11 +66,19 @@ void InterpreterAlterQuery::parseAlter( command.type = AlterCommand::ADD; const auto & ast_col_decl = typeid_cast(*params.col_decl); - StringRange type_range = ast_col_decl.type->range; - String type_string = String(type_range.first, type_range.second - type_range.first); command.column_name = ast_col_decl.name; - command.data_type = data_type_factory.get(type_string); + if (ast_col_decl.type) + { + StringRange type_range = ast_col_decl.type->range; + String type_string(type_range.first, type_range.second - type_range.first); + command.data_type = data_type_factory.get(type_string); + } + if (ast_col_decl.default_expression) + { + command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier); + command.default_expression = setAlias(ast_col_decl.default_expression, ast_col_decl.name); + } if (params.column) command.after_column = typeid_cast(*params.column).name; @@ -92,11 +99,20 @@ void InterpreterAlterQuery::parseAlter( command.type = AlterCommand::MODIFY; const auto & ast_col_decl = typeid_cast(*params.col_decl); - StringRange type_range = ast_col_decl.type->range; - String type_string = String(type_range.first, type_range.second - type_range.first); command.column_name = ast_col_decl.name; - command.data_type = data_type_factory.get(type_string); + if (ast_col_decl.type) + { + StringRange type_range = ast_col_decl.type->range; + String type_string(type_range.first, type_range.second - type_range.first); + command.data_type = data_type_factory.get(type_string); + } + + if (ast_col_decl.default_expression) + { + command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier); + command.default_expression = setAlias(ast_col_decl.default_expression, ast_col_decl.name); + } out_alter_commands.push_back(command); } @@ -115,143 +131,14 @@ void InterpreterAlterQuery::parseAlter( } } -void InterpreterAlterQuery::validateColumnChanges(ASTAlterQuery::ParameterContainer & params_container, const StoragePtr & table, const Context & context) -{ - auto columns = table->getColumnsList(); - columns.insert(std::end(columns), std::begin(table->materialized_columns), std::end(table->materialized_columns)); - columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns)); - auto defaults = table->column_defaults; - - std::vector> defaulted_columns{}; - - ASTPtr default_expr_list{new ASTExpressionList}; - default_expr_list->children.reserve(table->column_defaults.size()); - - for (auto & params : params_container) - { - if (params.type == ASTAlterQuery::ADD_COLUMN || params.type == ASTAlterQuery::MODIFY_COLUMN) - { - auto & col_decl = typeid_cast(*params.col_decl); - const auto & column_name = col_decl.name; - - if (params.type == ASTAlterQuery::MODIFY_COLUMN) - { - const auto it = std::find_if(std::begin(columns), std::end(columns), - std::bind(AlterCommand::namesEqual, std::cref(column_name), std::placeholders::_1)); - - if (it == std::end(columns)) - throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.", - DB::ErrorCodes::ILLEGAL_COLUMN); - - columns.erase(it); - defaults.erase(column_name); - } - - if (col_decl.type) - { - const StringRange & type_range = col_decl.type->range; - columns.emplace_back(col_decl.name, - context.getDataTypeFactory().get({type_range.first, type_range.second})); - } - - if (col_decl.default_expression) - { - - if (col_decl.type) - { - const auto tmp_column_name = col_decl.name + "_tmp"; - const auto & final_column_name = col_decl.name; - const auto conversion_function_name = "to" + columns.back().type->getName(); - - default_expr_list->children.emplace_back(setAlias( - makeASTFunction(conversion_function_name, ASTPtr{new ASTIdentifier{{}, tmp_column_name}}), - final_column_name)); - - default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), tmp_column_name)); - - defaulted_columns.emplace_back(columns.back().type.get(), &col_decl); - } - else - { - default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), col_decl.name)); - - defaulted_columns.emplace_back(nullptr, &col_decl); - } - } - } - else if (params.type == ASTAlterQuery::DROP_COLUMN) - { - const auto & column_name = typeid_cast(*(params.column)).name; - - auto found = false; - for (auto it = std::begin(columns); it != std::end(columns);) - if (AlterCommand::namesEqual(column_name, *it)) - { - found = true; - it = columns.erase(it); - } - else - ++it; - - for (auto it = std::begin(defaults); it != std::end(defaults);) - if (AlterCommand::namesEqual(column_name, { it->first, nullptr })) - it = defaults.erase(it); - else - ++it; - - if (!found) - throw Exception("Wrong column name. Cannot find column " + column_name + " to drop.", - DB::ErrorCodes::ILLEGAL_COLUMN); - } - } - - for (const auto & col_def : defaults) - default_expr_list->children.emplace_back(setAlias(col_def.second.expression->clone(), col_def.first)); - - const auto actions = ExpressionAnalyzer{default_expr_list, context, columns}.getActions(true); - const auto block = actions->getSampleBlock(); - - for (auto & column : defaulted_columns) - { - const auto type_ptr = column.first; - const auto col_decl_ptr = column.second; - - if (type_ptr) - { - const auto & tmp_column = block.getByName(col_decl_ptr->name + "_tmp"); - - /// type mismatch between explicitly specified and deduced type, add conversion - if (typeid(*type_ptr) != typeid(*tmp_column.type)) - { - col_decl_ptr->default_expression = makeASTFunction( - "to" + type_ptr->getName(), - col_decl_ptr->default_expression); - - col_decl_ptr->children.clear(); - col_decl_ptr->children.push_back(col_decl_ptr->type); - col_decl_ptr->children.push_back(col_decl_ptr->default_expression); - } - } - else - { - col_decl_ptr->type = new ASTIdentifier{}; - col_decl_ptr->query_string = new String{block.getByName(col_decl_ptr->name).type->getName()}; - col_decl_ptr->range = { - col_decl_ptr->query_string->data(), - col_decl_ptr->query_string->data() + col_decl_ptr->query_string->size() - }; - static_cast(*col_decl_ptr->type).name = *col_decl_ptr->query_string; - } - - defaults.emplace(col_decl_ptr->name, ColumnDefault{ - columnDefaultTypeFromString(col_decl_ptr->default_specifier), - setAlias(col_decl_ptr->default_expression, col_decl_ptr->name) - }); - } -} - void InterpreterAlterQuery::updateMetadata( - const String & database_name, const String & table_name, const NamesAndTypesList & columns, Context & context) + const String & database_name, + const String & table_name, + const NamesAndTypesList & columns, + const NamesAndTypesList & materialized_columns, + const NamesAndTypesList & alias_columns, + const ColumnDefaults & column_defaults, + Context & context) { String path = context.getPath(); @@ -286,7 +173,7 @@ void InterpreterAlterQuery::updateMetadata( ASTCreateQuery & attach = typeid_cast(*ast); - ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns); + ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns, materialized_columns, alias_columns, column_defaults); *std::find(attach.children.begin(), attach.children.end(), attach.columns) = new_columns; attach.columns = new_columns; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 1192c4908f9..5564de6774d 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -128,7 +128,7 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) } else if (!create.as_table.empty()) { - columns = new NamesAndTypesList(as_storage->getColumnsListAsterisk()); + columns = new NamesAndTypesList(as_storage->getColumnsListNonMaterialized()); materialized_columns = as_storage->materialized_columns; alias_columns = as_storage->alias_columns; column_defaults = as_storage->column_defaults; diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 0e24558adb4..161e476492b 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -67,7 +67,7 @@ void InterpreterSelectQuery::init(BlockInputStreamPtr input_, const NamesAndType table_lock = storage->lockStructure(false); if (table_column_names.empty()) - context.setColumns(storage->getColumnsListAsterisk()); + context.setColumns(storage->getColumnsListNonMaterialized()); } if (!table_column_names.empty()) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp new file mode 100644 index 00000000000..1bae4079ac1 --- /dev/null +++ b/dbms/src/Storages/AlterCommands.cpp @@ -0,0 +1,151 @@ +#include +#include +#include + +namespace DB +{ + void AlterCommand::apply(NamesAndTypesList & columns, + NamesAndTypesList & materialized_columns, + NamesAndTypesList & alias_columns, + ColumnDefaults & column_defaults) const + { + if (type == ADD) + { + const auto throw_if_exists = [this] (const NamesAndTypesList & columns) { + if (std::count_if(columns.begin(), columns.end(), + std::bind(namesEqual, std::cref(column_name), std::placeholders::_1))) + { + throw Exception("Cannot add column " + column_name + ": column with this name already exisits.", + DB::ErrorCodes::ILLEGAL_COLUMN); + } + }; + + throw_if_exists(columns); + throw_if_exists(materialized_columns); + throw_if_exists(alias_columns); + + const auto add_column = [this] (NamesAndTypesList & columns) { + auto insert_it = columns.end(); + + if (!after_column.empty()) + { + /// Пытаемся найти первую с конца колонку с именем column_name или с именем, начинающимся с column_name и ".". + /// Например "fruits.bananas" + /// одинаковыми считаются имена, если они совпадают целиком или name_without_dot совпадает с частью имени до точки + const auto reverse_insert_it = std::find_if(columns.rbegin(), columns.rend(), + std::bind(namesEqual, std::cref(after_column), std::placeholders::_1)); + + if (reverse_insert_it == columns.rend()) + throw Exception("Wrong column name. Cannot find column " + column_name + " to insert after", + DB::ErrorCodes::ILLEGAL_COLUMN); + else + { + /// base возвращает итератор, уже смещенный на один элемент вправо + insert_it = reverse_insert_it.base(); + } + } + + columns.emplace(insert_it, column_name, data_type); + }; + + if (default_type == ColumnDefaultType::Default) + add_column(columns); + else if (default_type == ColumnDefaultType::Materialized) + add_column(materialized_columns); + else if (default_type == ColumnDefaultType::Alias) + add_column(alias_columns); + else + throw Exception{"Unknown ColumnDefaultType value", ErrorCodes::LOGICAL_ERROR}; + + if (default_expression) + column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); + + /// Медленно, так как каждый раз копируется список + columns = *DataTypeNested::expandNestedColumns(columns); + } + else if (type == DROP) + { + /// look for a column in list and remove it if present, also removing corresponding entry from column_defaults + const auto remove_column = [&column_defaults, this] (NamesAndTypesList & columns) { + auto removed = false; + NamesAndTypesList::iterator column_it; + + while (columns.end() != (column_it = std::find_if(columns.begin(), columns.end(), + std::bind(namesEqual, std::cref(column_name), std::placeholders::_1)))) + { + removed = true; + column_it = columns.erase(column_it); + column_defaults.erase(column_name); + } + + return removed; + }; + + if (!remove_column(columns) && + !remove_column(materialized_columns) && + !remove_column(alias_columns)) + { + throw Exception("Wrong column name. Cannot find column " + column_name + " to drop", + DB::ErrorCodes::ILLEGAL_COLUMN); + } + } + else if (type == MODIFY) + { + const auto it = column_defaults.find(column_name); + const auto had_default_expr = it != column_defaults.end(); + const auto old_default_type = had_default_expr ? it->second.type : ColumnDefaultType{}; + + if (old_default_type != default_type) + throw Exception{"Cannot change column default specifier from " + toString(old_default_type) + + " to " + toString(default_type), 0 }; + + /// find column or throw exception + const auto find_column = [this] (NamesAndTypesList & columns) { + const auto it = std::find_if(columns.begin(), columns.end(), + std::bind(namesEqual, std::cref(column_name), std::placeholders::_1) ); + if (it == columns.end()) + throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.", + DB::ErrorCodes::ILLEGAL_COLUMN); + + return it; + }; + + /// find column in one of three column lists + const auto column_it = find_column( + default_type == ColumnDefaultType::Default ? columns : + default_type == ColumnDefaultType::Materialized ? materialized_columns : + alias_columns); + + column_it->type = data_type; + + /// remove, add or update default_expression + if (!default_expression && had_default_expr) + column_defaults.erase(column_name); + else if (default_expression && !had_default_expr) + column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); + else + column_defaults[column_name].expression = default_expression; + } + else + throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR); + } + + void AlterCommands::apply(NamesAndTypesList & columns, + NamesAndTypesList & materialized_columns, + NamesAndTypesList & alias_columns, + ColumnDefaults & column_defaults) const + { + auto new_columns = columns; + auto new_materialized_columns = materialized_columns; + auto new_alias_columns = alias_columns; + auto new_column_defaults = column_defaults; + + for (const AlterCommand & command : *this) + command.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); + + columns = std::move(new_columns); + materialized_columns = std::move(new_materialized_columns); + alias_columns = std::move(new_alias_columns); + column_defaults = std::move(new_column_defaults); + } +} diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index 0a6144b20b8..2621180b24c 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -103,7 +103,7 @@ Block ITableDeclaration::getSampleBlockNonMaterialized() const { Block res; - for (const auto & col : getColumnsListAsterisk()) + for (const auto & col : getColumnsListNonMaterialized()) res.insert({ col.type->createColumn(), col.type, col.name }); return res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index ba3eb9e25ee..ff1113afd98 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -360,8 +360,11 @@ void MergeTreeData::dropAllData() void MergeTreeData::checkAlter(const AlterCommands & params) { /// Проверим, что указанные преобразования можно совершить над списком столбцов без учета типов. - NamesAndTypesList new_columns = *columns; - params.apply(new_columns); + auto new_columns = *columns; + auto new_materialized_columns = materialized_columns; + auto new_alias_columns = alias_columns; + auto new_column_defaults = column_defaults; + params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); /// Список столбцов, которые нельзя трогать. /// sampling_expression можно не учитывать, потому что он обязан содержаться в первичном ключе. diff --git a/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp b/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp index ef1b36776fd..69c124e3675 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 98d0785907e..f1701bdefb4 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -203,8 +203,9 @@ BlockOutputStreamPtr StorageDistributed::write(ASTPtr query) void StorageDistributed::alter(const AlterCommands & params, const String & database_name, const String & table_name, Context & context) { auto lock = lockStructureForAlter(); - params.apply(*columns); - InterpreterAlterQuery::updateMetadata(database_name, table_name, *columns, context); + params.apply(*columns, materialized_columns, alias_columns, column_defaults); + InterpreterAlterQuery::updateMetadata(database_name, table_name, + *columns, materialized_columns, alias_columns, column_defaults, context); } void StorageDistributed::shutdown() diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 13409135104..a3d03eb6f14 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -203,8 +203,9 @@ void StorageMerge::getSelectedTables(StorageVector & selected_tables) const void StorageMerge::alter(const AlterCommands & params, const String & database_name, const String & table_name, Context & context) { auto lock = lockStructureForAlter(); - params.apply(*columns); - InterpreterAlterQuery::updateMetadata(database_name, table_name, *columns, context); + params.apply(*columns, materialized_columns, alias_columns, column_defaults); + InterpreterAlterQuery::updateMetadata(database_name, table_name, *columns, + materialized_columns, alias_columns, column_defaults, context); } } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 9807e1738f8..ec05541a8f2 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -129,8 +129,12 @@ void StorageMergeTree::alter(const AlterCommands & params, const String & databa data.checkAlter(params); - NamesAndTypesList new_columns = data.getColumnsList(); - params.apply(new_columns); + auto new_columns = data.getColumnsListNonMaterialized(); + auto new_materialized_columns = data.materialized_columns; + auto new_alias_columns = data.alias_columns; + auto new_column_defaults = data.column_defaults; + + params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); MergeTreeData::DataParts parts = data.getDataParts(); std::vector transactions; @@ -143,8 +147,17 @@ void StorageMergeTree::alter(const AlterCommands & params, const String & databa auto table_hard_lock = lockStructureForAlter(); - InterpreterAlterQuery::updateMetadata(database_name, table_name, new_columns, context); + InterpreterAlterQuery::updateMetadata(database_name, table_name, new_columns, + new_materialized_columns, new_alias_columns, new_column_defaults, context); + + materialized_columns = new_materialized_columns; + alias_columns = new_alias_columns; + column_defaults = new_column_defaults; + data.setColumnsList(new_columns); + data.materialized_columns = std::move(new_materialized_columns); + data.alias_columns = std::move(new_alias_columns); + data.column_defaults = std::move(new_column_defaults); for (auto & transaction : transactions) { diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 088f302b57d..e5f21ee1b86 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -207,7 +207,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists() zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); ops.push_back(new zkutil::Op::Create(zookeeper_path + "/metadata", metadata.str(), zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); - ops.push_back(new zkutil::Op::Create(zookeeper_path + "/columns", data.getColumnsList().toString(), + ops.push_back(new zkutil::Op::Create(zookeeper_path + "/columns", data.getColumnsListNonMaterialized().toString(), zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); ops.push_back(new zkutil::Op::Create(zookeeper_path + "/log", "", zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); @@ -256,16 +256,23 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo zkutil::Stat stat; auto columns = NamesAndTypesList::parse(zookeeper->get(zookeeper_path + "/columns", &stat), context.getDataTypeFactory()); + NamesAndTypesList materialized_columns; + NamesAndTypesList alias_columns; + ColumnDefaults column_defaults; columns_version = stat.version; - if (columns != data.getColumnsList()) + if (columns != data.getColumnsListNonMaterialized()) { - if (allow_alter && (data.getColumnsList().sizeOfDifference(columns) <= 2 || skip_sanity_checks)) + if (allow_alter && (data.getColumnsListNonMaterialized().sizeOfDifference(columns) <= 2 || skip_sanity_checks)) { LOG_WARNING(log, "Table structure in ZooKeeper is a little different from local table structure. Assuming ALTER."); /// Без всяких блокировок, потому что таблица еще не создана. - InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, context); + InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, + materialized_columns, alias_columns, column_defaults, context); data.setColumnsList(columns); + data.materialized_columns = std::move(materialized_columns); + data.alias_columns = std::move(alias_columns); + data.column_defaults = std::move(column_defaults); } else { @@ -403,7 +410,7 @@ void StorageReplicatedMergeTree::createReplica() LOG_DEBUG(log, "Copied " << source_queue.size() << " queue entries"); } - zookeeper->create(replica_path + "/columns", data.getColumnsList().toString(), zkutil::CreateMode::Persistent); + zookeeper->create(replica_path + "/columns", data.getColumnsListNonMaterialized().toString(), zkutil::CreateMode::Persistent); } void StorageReplicatedMergeTree::activateReplica() @@ -1583,6 +1590,9 @@ void StorageReplicatedMergeTree::alterThread() zkutil::Stat stat; String columns_str = zookeeper->get(zookeeper_path + "/columns", &stat, alter_thread_event); NamesAndTypesList columns = NamesAndTypesList::parse(columns_str, context.getDataTypeFactory()); + NamesAndTypesList materialized_columns; + NamesAndTypesList alias_columns; + ColumnDefaults column_defaults; bool changed_version = (stat.version != columns_version); @@ -1593,12 +1603,22 @@ void StorageReplicatedMergeTree::alterThread() { auto table_lock = lockStructureForAlter(); - if (columns != data.getColumnsList()) + if (columns != data.getColumnsListNonMaterialized()) { LOG_INFO(log, "Columns list changed in ZooKeeper. Applying changes locally."); - InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, context); + InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, + materialized_columns, alias_columns, column_defaults, context); + + this->materialized_columns = materialized_columns; + this->alias_columns = alias_columns; + this->column_defaults = column_defaults; + data.setColumnsList(columns); + data.materialized_columns = std::move(materialized_columns); + data.alias_columns = std::move(alias_columns); + data.column_defaults = std::move(column_defaults); + if (unreplicated_data) unreplicated_data->setColumnsList(columns); LOG_INFO(log, "Applied changes to table."); @@ -2306,6 +2326,9 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, LOG_DEBUG(log, "Doing ALTER"); NamesAndTypesList new_columns; + NamesAndTypesList new_materialized_columns; + NamesAndTypesList new_alias_columns; + ColumnDefaults new_column_defaults; String new_columns_str; int new_columns_version; zkutil::Stat stat; @@ -2318,8 +2341,11 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, data.checkAlter(params); - new_columns = data.getColumnsList(); - params.apply(new_columns); + new_columns = data.getColumnsListNonMaterialized(); + new_materialized_columns = data.materialized_columns; + new_alias_columns = data.alias_columns; + new_column_defaults = data.column_defaults; + params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); new_columns_str = new_columns.toString(); diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 156385a61c1..bb059a299da 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -36,7 +36,7 @@ StorageView::StorageView( const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_) - : IStorage{materialized_columns_, alias_columns_, column_defaults}, table_name(table_name_), + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, table_name(table_name_), database_name(database_name_), context(context_), columns(columns_) { ASTCreateQuery & create = typeid_cast(*query_); From 1dfcaefc64f6ea7f772ffc1925362eb185501419 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 16 Oct 2014 20:55:23 +0400 Subject: [PATCH 16/29] Merge --- dbms/include/DB/Core/Block.h | 8 ++++--- .../AddingDefaultBlockInputStream.h | 2 +- .../AddingDefaultBlockOutputStream.h | 2 +- .../DB/Storages/MergeTree/MergeTreeReader.h | 5 ++++- dbms/src/Core/Block.cpp | 22 ++++++++++--------- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/dbms/include/DB/Core/Block.h b/dbms/include/DB/Core/Block.h index cc6c0094bf0..667c6a3d7dd 100644 --- a/dbms/include/DB/Core/Block.h +++ b/dbms/include/DB/Core/Block.h @@ -65,9 +65,11 @@ public: /// удалить столбец с заданным именем void erase(const String & name); /// Добавляет в блок недостающие столбцы со значениями по-умолчанию - void addDefaults(NamesAndTypesListPtr required_columns); - void addDefaults(NamesAndTypesListPtr required_columns, - const ColumnDefaults & column_defaults, const Context & context); + void addDefaults(const NamesAndTypesList & required_columns); + void addDefaults(const NamesAndTypesList & required_columns, + const ColumnDefaults & column_defaults, + const Context & context, + bool only_explicitly_defaulted = false); ColumnWithNameAndType & getByPosition(size_t position); const ColumnWithNameAndType & getByPosition(size_t position) const; diff --git a/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h b/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h index 21648f69544..76b10b7d0e1 100644 --- a/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h +++ b/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h @@ -54,7 +54,7 @@ protected: Block res = children.back()->read(); if (!res) return res; - res.addDefaults(required_columns, column_defaults, context); + res.addDefaults(*required_columns, column_defaults, context); return res; } diff --git a/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h b/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h index 48e0450a6ea..3d2fd157f65 100644 --- a/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h +++ b/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h @@ -38,7 +38,7 @@ public: void write(const Block & block) override { Block res = block; - res.addDefaults(required_columns, column_defaults, context); + res.addDefaults(*required_columns, column_defaults, context); output->write(res); } diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h b/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h index a3f771b52c2..953ca94a88c 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h @@ -161,7 +161,7 @@ public: size_t pos = 0; /// Позиция, куда надо вставить недостающий столбец. for (NamesAndTypesList::const_iterator it = columns.begin(); it != columns.end(); ++it, ++pos) { - if (!res.has(it->name)) + if (!res.has(it->name) && storage.column_defaults.count(it->name) == 0) { ColumnWithNameAndType column; column.name = it->name; @@ -192,6 +192,9 @@ public: res.insert(pos, column); } } + + /// evaluate defaulted values + res.addDefaults(columns, storage.column_defaults, storage.context, true); } catch (const Exception & e) { diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index fdb22ac0c81..800cdd3fce2 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -25,30 +25,32 @@ Block::Block(const Block & other) } -void Block::addDefaults(NamesAndTypesListPtr required_columns) +void Block::addDefaults(const NamesAndTypesList & required_columns) { - for (const auto & column : *required_columns) + for (const auto & column : required_columns) if (!has(column.name)) insertDefault(column.name, column.type); } -void Block::addDefaults(NamesAndTypesListPtr required_columns, - const ColumnDefaults & column_defaults, const Context & context) +void Block::addDefaults(const NamesAndTypesList & required_columns, + const ColumnDefaults & column_defaults, + const Context & context, + const bool only_explicitly_defaulted) { ASTPtr default_expr_list{stdext::make_unique().release()}; - for (const auto & column : *required_columns) + for (const auto & column : required_columns) { if (has(column.name)) continue; const auto it = column_defaults.find(column.name); - /// expressions must be cloned to prevent modification by ExpressionAnalyzer - if (it == column_defaults.end()) - insertDefault(column.name, column.type); - else + /// expressions must be cloned to prevent modification by the ExpressionAnalyzer + if (it != column_defaults.end()) default_expr_list->children.emplace_back(it->second.expression->clone()); + else if (only_explicitly_defaulted) + insertDefault(column.name, column.type); } /// nothing to evaluate @@ -59,7 +61,7 @@ void Block::addDefaults(NamesAndTypesListPtr required_columns, * we are going to operate on a copy instead of the original block */ Block copy_block{*this}; /// evaluate default values for defaulted columns - ExpressionAnalyzer{default_expr_list, context, *required_columns}.getActions(true)->execute(copy_block); + ExpressionAnalyzer{default_expr_list, context, required_columns}.getActions(true)->execute(copy_block); /// move evaluated columns to the original block for (auto & column_name_type : copy_block.getColumns()) From 8a8e5963c5c9dc6692ca94f39670e42724f9b1ab Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Fri, 17 Oct 2014 16:26:53 +0400 Subject: [PATCH 17/29] dbms: chaining support for defaulted columns when corresponding columns files are absent. [#METR-12739] --- dbms/include/DB/Core/Block.h | 6 ++- .../DB/Storages/MergeTree/MergeTreeReader.h | 5 ++- dbms/src/Core/Block.cpp | 41 ++++++++++++++++--- dbms/src/Storages/AlterCommands.cpp | 2 +- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/dbms/include/DB/Core/Block.h b/dbms/include/DB/Core/Block.h index 667c6a3d7dd..1787e1bc2d7 100644 --- a/dbms/include/DB/Core/Block.h +++ b/dbms/include/DB/Core/Block.h @@ -68,8 +68,10 @@ public: void addDefaults(const NamesAndTypesList & required_columns); void addDefaults(const NamesAndTypesList & required_columns, const ColumnDefaults & column_defaults, - const Context & context, - bool only_explicitly_defaulted = false); + const Context & context); + void addAllDefaults(const NamesAndTypesList & required_columns, + const ColumnDefaults & column_defaults, + const Context & context); ColumnWithNameAndType & getByPosition(size_t position); const ColumnWithNameAndType & getByPosition(size_t position) const; diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h b/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h index 953ca94a88c..a97c2ef05ec 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h @@ -161,6 +161,7 @@ public: size_t pos = 0; /// Позиция, куда надо вставить недостающий столбец. for (NamesAndTypesList::const_iterator it = columns.begin(); it != columns.end(); ++it, ++pos) { + /// insert default values only for columns without default expressions if (!res.has(it->name) && storage.column_defaults.count(it->name) == 0) { ColumnWithNameAndType column; @@ -193,8 +194,8 @@ public: } } - /// evaluate defaulted values - res.addDefaults(columns, storage.column_defaults, storage.context, true); + /// evaluate defaulted columns + res.addAllDefaults(columns, storage.column_defaults, storage.context); } catch (const Exception & e) { diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index 800cdd3fce2..1ce86ed3858 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -34,8 +34,7 @@ void Block::addDefaults(const NamesAndTypesList & required_columns) void Block::addDefaults(const NamesAndTypesList & required_columns, const ColumnDefaults & column_defaults, - const Context & context, - const bool only_explicitly_defaulted) + const Context & context) { ASTPtr default_expr_list{stdext::make_unique().release()}; @@ -47,10 +46,10 @@ void Block::addDefaults(const NamesAndTypesList & required_columns, const auto it = column_defaults.find(column.name); /// expressions must be cloned to prevent modification by the ExpressionAnalyzer - if (it != column_defaults.end()) - default_expr_list->children.emplace_back(it->second.expression->clone()); - else if (only_explicitly_defaulted) + if (it == column_defaults.end()) insertDefault(column.name, column.type); + else + default_expr_list->children.emplace_back(it->second.expression->clone()); } /// nothing to evaluate @@ -68,6 +67,38 @@ void Block::addDefaults(const NamesAndTypesList & required_columns, insert(std::move(column_name_type)); } +void Block::addAllDefaults(const NamesAndTypesList & required_columns, + const ColumnDefaults & column_defaults, + const Context & context) +{ + ASTPtr default_expr_list{stdext::make_unique().release()}; + + for (const auto & column_default : column_defaults) + { + if (has(column_default.first)) + continue; + + /// expressions must be cloned to prevent modification by the ExpressionAnalyzer + default_expr_list->children.emplace_back(column_default.second.expression->clone()); + } + + /// nothing to evaluate + if (default_expr_list->children.empty()) + return; + + /** ExpressionAnalyzer eliminates "unused" columns, in order to ensure their safety + * we are going to operate on a copy instead of the original block */ + Block copy_block{*this}; + /// evaluate default values for defaulted columns + ExpressionAnalyzer{default_expr_list, context, required_columns}.getActions(true)->execute(copy_block); + + /// move evaluated columns to the original block only if required + for (auto & column_name_type : required_columns) { + if (copy_block.has(column_name_type.name)) + insert(std::move(copy_block.getByName(column_name_type.name))); + } +} + Block & Block::operator= (const Block & other) { data = other.data; diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 1bae4079ac1..824fb183a1f 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -123,7 +123,7 @@ namespace DB column_defaults.erase(column_name); else if (default_expression && !had_default_expr) column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); - else + else if (had_default_expr) column_defaults[column_name].expression = default_expression; } else From 847b91614a63c86ad3ce0bdabad86204a4bc8a8d Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 21 Oct 2014 16:11:20 +0400 Subject: [PATCH 18/29] dbms: deduce and check types during alter, alteration of defaulted columns. [#METR-12739] --- dbms/include/DB/Core/Block.h | 5 +- .../Interpreters/InterpreterDescribeQuery.h | 3 +- dbms/include/DB/Storages/AlterCommands.h | 13 ++ dbms/include/DB/Storages/ColumnDefault.h | 15 +- dbms/include/DB/Storages/IStorage.h | 9 +- .../MergeTree/MergeTreeBlockInputStream.h | 1 + .../DB/Storages/MergeTree/MergeTreeReader.h | 7 +- dbms/src/Core/Block.cpp | 37 +---- .../Interpreters/InterpreterAlterQuery.cpp | 12 +- .../Interpreters/InterpreterCreateQuery.cpp | 4 +- dbms/src/Storages/AlterCommands.cpp | 156 +++++++++++++++++- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 6 +- dbms/src/Storages/StorageMergeTree.cpp | 9 +- .../Storages/StorageReplicatedMergeTree.cpp | 108 +++++++++--- 14 files changed, 305 insertions(+), 80 deletions(-) diff --git a/dbms/include/DB/Core/Block.h b/dbms/include/DB/Core/Block.h index 1787e1bc2d7..05a8d5a47de 100644 --- a/dbms/include/DB/Core/Block.h +++ b/dbms/include/DB/Core/Block.h @@ -9,7 +9,6 @@ #include #include #include -#include #include "ColumnsWithNameAndType.h" @@ -22,6 +21,7 @@ namespace DB */ class Context; +struct ColumnDefaults; class Block { @@ -69,9 +69,6 @@ public: void addDefaults(const NamesAndTypesList & required_columns, const ColumnDefaults & column_defaults, const Context & context); - void addAllDefaults(const NamesAndTypesList & required_columns, - const ColumnDefaults & column_defaults, - const Context & context); ColumnWithNameAndType & getByPosition(size_t position); const ColumnWithNameAndType & getByPosition(size_t position) const; diff --git a/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h b/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h index 2f94fe854e6..4b91eea4cc5 100644 --- a/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterDescribeQuery.h @@ -110,8 +110,7 @@ private: else { default_type_column.column->insert(toString(it->second.type)); - default_expression_column.column->insert(queryToString( - setAlias(it->second.expression->clone(), ""))); + default_expression_column.column->insert(queryToString(it->second.expression)); } } diff --git a/dbms/include/DB/Storages/AlterCommands.h b/dbms/include/DB/Storages/AlterCommands.h index 8f67c021e53..ce2256706e7 100644 --- a/dbms/include/DB/Storages/AlterCommands.h +++ b/dbms/include/DB/Storages/AlterCommands.h @@ -41,8 +41,19 @@ struct AlterCommand NamesAndTypesList & materialized_columns, NamesAndTypesList & alias_columns, ColumnDefaults & column_defaults) const; + + AlterCommand() = default; + AlterCommand(const Type type, const String & column_name, const DataTypePtr & data_type, + const ColumnDefaultType default_type, const ASTPtr & default_expression, + const String & after_column = String{}) + : type{type}, column_name{column_name}, data_type{data_type}, default_type{default_type}, + default_expression{default_expression}, after_column{after_column} + {} }; +class IStorage; +class Context; + class AlterCommands : public std::vector { public: @@ -50,6 +61,8 @@ public: NamesAndTypesList & materialized_columns, NamesAndTypesList & alias_columns, ColumnDefaults & column_defaults) const; + + void validate(IStorage * table, const Context & context); }; } diff --git a/dbms/include/DB/Storages/ColumnDefault.h b/dbms/include/DB/Storages/ColumnDefault.h index 68b667fea26..4da32b40dc3 100644 --- a/dbms/include/DB/Storages/ColumnDefault.h +++ b/dbms/include/DB/Storages/ColumnDefault.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include namespace DB @@ -56,5 +57,17 @@ namespace DB ASTPtr expression; }; - using ColumnDefaults = std::unordered_map; + inline bool operator==(const ColumnDefault & lhs, const ColumnDefault & rhs) + { + return lhs.type == rhs.type && queryToString(lhs.expression) == queryToString(rhs.expression); + } + + struct ColumnDefaults : public std::unordered_map + { + using std::unordered_map::unordered_map; + + /// @todo implement (de)serialization + String toString() const { return {}; } + static ColumnDefaults parse(const String & str) { return {}; } + }; } diff --git a/dbms/include/DB/Storages/IStorage.h b/dbms/include/DB/Storages/IStorage.h index f086ae8abd7..8de35859a4b 100644 --- a/dbms/include/DB/Storages/IStorage.h +++ b/dbms/include/DB/Storages/IStorage.h @@ -14,6 +14,7 @@ #include #include #include +#include namespace DB @@ -107,8 +108,8 @@ public: return res; } - typedef Poco::SharedPtr TableStructureWriteLockPtr; - typedef Poco::SharedPtr TableDataWriteLockPtr; + typedef std::unique_ptr TableStructureWriteLockPtr; + typedef std::unique_ptr TableDataWriteLockPtr; typedef std::pair TableFullWriteLockPtr; /** Не дает читать структуру таблицы. Берется для ALTER, RENAME и DROP. @@ -124,7 +125,7 @@ public: */ TableDataWriteLockPtr lockDataForAlter() { - TableDataWriteLockPtr res = new Poco::ScopedWriteRWLock(data_lock); + auto res = stdext::make_unique(data_lock); if (is_dropped) throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED); return res; @@ -132,7 +133,7 @@ public: TableStructureWriteLockPtr lockStructureForAlter() { - TableStructureWriteLockPtr res = new Poco::ScopedWriteRWLock(structure_lock); + auto res = stdext::make_unique(structure_lock); if (is_dropped) throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED); return res; diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h b/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h index 64396a5aeef..ff8e7254cfe 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h @@ -108,6 +108,7 @@ protected: if (!reader) { + /// @todo resolve missing columns UncompressedCache * uncompressed_cache = use_uncompressed_cache ? storage.context.getUncompressedCache() : NULL; reader.reset(new MergeTreeReader(path, owned_data_part->name, columns, uncompressed_cache, storage, all_mark_ranges)); if (prewhere_actions) diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h b/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h index a97c2ef05ec..a72086c1f4d 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h @@ -124,7 +124,7 @@ public: storage.reportBrokenPart(part_name); /// Более хорошая диагностика. - throw Exception(e.message() + " (while reading from part " + path + " from mark " + toString(from_mark) + " to " + throw Exception(e.message() + "\n(while reading from part " + path + " from mark " + toString(from_mark) + " to " + toString(to_mark) + ")", e.code()); } catch (...) @@ -195,12 +195,13 @@ public: } /// evaluate defaulted columns - res.addAllDefaults(columns, storage.column_defaults, storage.context); + res.addDefaults(columns, storage.column_defaults, storage.context); } catch (const Exception & e) { /// Более хорошая диагностика. - throw Exception(e.message() + " (while reading from part " + path + ")", e.code()); + throw Exception(e.message() + '\n' + e.getStackTrace().toString() + + "\n(while reading from part " + path + ")", e.code()); } } diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index 1ce86ed3858..47f3da4165e 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -5,6 +5,8 @@ #include +#include + #include #include @@ -49,7 +51,8 @@ void Block::addDefaults(const NamesAndTypesList & required_columns, if (it == column_defaults.end()) insertDefault(column.name, column.type); else - default_expr_list->children.emplace_back(it->second.expression->clone()); + default_expr_list->children.emplace_back( + setAlias(it->second.expression->clone(), it->first)); } /// nothing to evaluate @@ -67,38 +70,6 @@ void Block::addDefaults(const NamesAndTypesList & required_columns, insert(std::move(column_name_type)); } -void Block::addAllDefaults(const NamesAndTypesList & required_columns, - const ColumnDefaults & column_defaults, - const Context & context) -{ - ASTPtr default_expr_list{stdext::make_unique().release()}; - - for (const auto & column_default : column_defaults) - { - if (has(column_default.first)) - continue; - - /// expressions must be cloned to prevent modification by the ExpressionAnalyzer - default_expr_list->children.emplace_back(column_default.second.expression->clone()); - } - - /// nothing to evaluate - if (default_expr_list->children.empty()) - return; - - /** ExpressionAnalyzer eliminates "unused" columns, in order to ensure their safety - * we are going to operate on a copy instead of the original block */ - Block copy_block{*this}; - /// evaluate default values for defaulted columns - ExpressionAnalyzer{default_expr_list, context, required_columns}.getActions(true)->execute(copy_block); - - /// move evaluated columns to the original block only if required - for (auto & column_name_type : required_columns) { - if (copy_block.has(column_name_type.name)) - insert(std::move(copy_block.getByName(column_name_type.name))); - } -} - Block & Block::operator= (const Block & other) { data = other.data; diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 5ebe338d5f8..b86da11bfdb 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -50,8 +50,12 @@ void InterpreterAlterQuery::execute() throw Exception("Bad PartitionCommand::Type: " + toString(command.type), ErrorCodes::ARGUMENT_OUT_OF_BOUND); } - if (!alter_commands.empty()) - table->alter(alter_commands, database_name, table_name, context); + if (alter_commands.empty()) + return; + + alter_commands.validate(table.get(), context); + + table->alter(alter_commands, database_name, table_name, context); } void InterpreterAlterQuery::parseAlter( @@ -77,7 +81,7 @@ void InterpreterAlterQuery::parseAlter( if (ast_col_decl.default_expression) { command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier); - command.default_expression = setAlias(ast_col_decl.default_expression, ast_col_decl.name); + command.default_expression = ast_col_decl.default_expression; } if (params.column) @@ -111,7 +115,7 @@ void InterpreterAlterQuery::parseAlter( if (ast_col_decl.default_expression) { command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier); - command.default_expression = setAlias(ast_col_decl.default_expression, ast_col_decl.name); + command.default_expression = ast_col_decl.default_expression; } out_alter_commands.push_back(command); diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 5564de6774d..5d5e3f31c50 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -337,7 +337,7 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( defaults.emplace(col_decl_ptr->name, ColumnDefault{ columnDefaultTypeFromString(col_decl_ptr->default_specifier), - setAlias(col_decl_ptr->default_expression, col_decl_ptr->name) + col_decl_ptr->default_expression }); } } @@ -429,7 +429,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(NamesAndTypesList columns, if (it != std::end(column_defaults)) { column_declaration->default_specifier = toString(it->second.type); - column_declaration->default_expression = setAlias(it->second.expression->clone(), ""); + column_declaration->default_expression = it->second.expression->clone(); } columns_list.children.push_back(column_declaration_ptr); diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 824fb183a1f..4bd280fdfbf 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -1,6 +1,9 @@ #include +#include #include #include +#include +#include namespace DB { @@ -54,7 +57,7 @@ namespace DB add_column(materialized_columns); else if (default_type == ColumnDefaultType::Alias) add_column(alias_columns); - else + else throw Exception{"Unknown ColumnDefaultType value", ErrorCodes::LOGICAL_ERROR}; if (default_expression) @@ -139,7 +142,7 @@ namespace DB auto new_materialized_columns = materialized_columns; auto new_alias_columns = alias_columns; auto new_column_defaults = column_defaults; - + for (const AlterCommand & command : *this) command.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); @@ -148,4 +151,153 @@ namespace DB alias_columns = std::move(new_alias_columns); column_defaults = std::move(new_column_defaults); } + + void AlterCommands::validate(IStorage * table, const Context & context) + { + auto lock = table->lockDataForAlter(); + + auto columns = table->getColumnsList(); + columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns)); + auto defaults = table->column_defaults; + + lock.reset(); + + std::vector> defaulted_columns{}; + + ASTPtr default_expr_list{new ASTExpressionList}; + default_expr_list->children.reserve(defaults.size()); + + for (AlterCommand & command : *this) + { + if (command.type == AlterCommand::ADD || command.type == AlterCommand::MODIFY) + { + if (command.type == AlterCommand::MODIFY) + { + const auto it = std::find_if(std::begin(columns), std::end(columns), + std::bind(AlterCommand::namesEqual, std::cref(command.column_name), std::placeholders::_1)); + + if (it == std::end(columns)) + throw Exception("Wrong column name. Cannot find column " + command.column_name + " to modify.", + DB::ErrorCodes::ILLEGAL_COLUMN); + + columns.erase(it); + defaults.erase(command.column_name); + } + + if (command.data_type) + columns.emplace_back(command.column_name, command.data_type); + + if (command.default_expression) + { + if (command.data_type) + { + const auto & column_name = command.column_name; + const auto tmp_column_name = column_name + "_tmp"; + const auto conversion_function_name = "to" + command.data_type->getName(); + + default_expr_list->children.emplace_back(setAlias( + makeASTFunction(conversion_function_name, ASTPtr{new ASTIdentifier{{}, tmp_column_name}}), + column_name)); + + default_expr_list->children.emplace_back(setAlias(command.default_expression->clone(), tmp_column_name)); + + defaulted_columns.emplace_back(command.column_name, &command); + } + else + { + default_expr_list->children.emplace_back( + setAlias(command.default_expression->clone(), command.column_name)); + + defaulted_columns.emplace_back(command.column_name, &command); + } + } + } + else if (command.type == AlterCommand::DROP) + { + auto found = false; + for (auto it = std::begin(columns); it != std::end(columns);) + if (AlterCommand::namesEqual(command.column_name, *it)) + { + found = true; + it = columns.erase(it); + } + else + ++it; + + for (auto it = std::begin(defaults); it != std::end(defaults);) + if (AlterCommand::namesEqual(command.column_name, { it->first, nullptr })) + it = defaults.erase(it); + else + ++it; + + if (!found) + throw Exception("Wrong column name. Cannot find column " + command.column_name + " to drop.", + DB::ErrorCodes::ILLEGAL_COLUMN); + } + } + + /** Existing defaulted columns may require default expression extensions with a type conversion, + * therefore we add them to defaulted_columns to allow further processing */ + for (const auto & col_def : defaults) + { + const auto & column_name = col_def.first; + const auto column_it = std::find_if(columns.begin(), columns.end(), [&] (const NameAndTypePair & name_type) { + return AlterCommand::namesEqual(column_name, name_type); + }); + const auto tmp_column_name = column_name + "_tmp"; + const auto conversion_function_name = "to" + column_it->type->getName(); + + default_expr_list->children.emplace_back(setAlias( + makeASTFunction(conversion_function_name, ASTPtr{new ASTIdentifier{{}, tmp_column_name}}), + column_name)); + + default_expr_list->children.emplace_back(setAlias(col_def.second.expression->clone(), tmp_column_name)); + + defaulted_columns.emplace_back(column_name, nullptr); + } + + const auto actions = ExpressionAnalyzer{default_expr_list, context, columns}.getActions(true); + const auto block = actions->getSampleBlock(); + + /// set deduced types, modify default expression if necessary + for (auto & defaulted_column : defaulted_columns) + { + const auto & column_name = defaulted_column.first; + const auto command_ptr = defaulted_column.second; + const auto & column = block.getByName(column_name); + + /// default expression on old column + if (!command_ptr) + { + const auto & tmp_column = block.getByName(column_name + "_tmp"); + + // column not specified explicitly in the ALTER query may require default_expression modification + if (typeid(*column.type) != typeid(*tmp_column.type)) + { + const auto it = defaults.find(column_name); + this->push_back(AlterCommand{ + AlterCommand::MODIFY, column_name, column.type, it->second.type, + makeASTFunction("to" + column.type->getName(), it->second.expression), + }); + } + } + else if (command_ptr && command_ptr->data_type) + { + const auto & tmp_column = block.getByName(column_name + "_tmp"); + + /// type mismatch between explicitly specified and deduced type, add conversion + if (typeid(*column.type) != typeid(*tmp_column.type)) + { + command_ptr->default_expression = makeASTFunction( + "to" + column.type->getName(), + command_ptr->default_expression->clone()); + } + } + else + { + /// just set deduced type + command_ptr->data_type = column.type; + } + } + } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index ff1113afd98..4cab03b3eb9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -381,7 +381,11 @@ void MergeTreeData::checkAlter(const AlterCommands & params) /// Проверим, что преобразования типов возможны. ExpressionActionsPtr unused_expression; NameToNameMap unused_map; - createConvertExpression(nullptr, *columns, new_columns, unused_expression, unused_map); + + /// augment plain columns with materialized columns for convert expression creation + new_columns.insert(std::end(new_columns), + std::begin(new_materialized_columns), std::end(new_materialized_columns)); + createConvertExpression(nullptr, getColumnsList(), new_columns, unused_expression, unused_map); } void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index ec05541a8f2..a3e23b0110b 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -133,15 +133,18 @@ void StorageMergeTree::alter(const AlterCommands & params, const String & databa auto new_materialized_columns = data.materialized_columns; auto new_alias_columns = data.alias_columns; auto new_column_defaults = data.column_defaults; - + params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); + auto columns_for_parts = new_columns; + columns_for_parts.insert(std::end(columns_for_parts), + std::begin(new_materialized_columns), std::end(new_materialized_columns)); + MergeTreeData::DataParts parts = data.getDataParts(); std::vector transactions; for (const MergeTreeData::DataPartPtr & part : parts) { - auto transaction = data.alterDataPart(part, new_columns); - if (transaction) + if (auto transaction = data.alterDataPart(part, columns_for_parts)) transactions.push_back(std::move(transaction)); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index e5f21ee1b86..aaea1e3cc96 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -209,6 +209,12 @@ void StorageReplicatedMergeTree::createTableIfNotExists() zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); ops.push_back(new zkutil::Op::Create(zookeeper_path + "/columns", data.getColumnsListNonMaterialized().toString(), zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); + ops.push_back(new zkutil::Op::Create(zookeeper_path + "/materialized_columns", data.materialized_columns.toString(), + zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); + ops.push_back(new zkutil::Op::Create(zookeeper_path + "/alias_columns", data.alias_columns.toString(), + zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); + ops.push_back(new zkutil::Op::Create(zookeeper_path + "/column_defaults", data.column_defaults.toString(), + zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); ops.push_back(new zkutil::Op::Create(zookeeper_path + "/log", "", zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); ops.push_back(new zkutil::Op::Create(zookeeper_path + "/blocks", "", @@ -256,13 +262,21 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo zkutil::Stat stat; auto columns = NamesAndTypesList::parse(zookeeper->get(zookeeper_path + "/columns", &stat), context.getDataTypeFactory()); - NamesAndTypesList materialized_columns; - NamesAndTypesList alias_columns; - ColumnDefaults column_defaults; + auto materialized_columns = NamesAndTypesList::parse( + zookeeper->get(zookeeper_path + "/materialized_columns", &stat), context.getDataTypeFactory()); + auto alias_columns = NamesAndTypesList::parse( + zookeeper->get(zookeeper_path + "/alias_columns", &stat), context.getDataTypeFactory()); + auto column_defaults = ColumnDefaults::parse(zookeeper->get(zookeeper_path + "/column_defaults", &stat)); columns_version = stat.version; - if (columns != data.getColumnsListNonMaterialized()) + if (columns != data.getColumnsListNonMaterialized() || + materialized_columns != data.materialized_columns || + alias_columns != data.alias_columns || + column_defaults != data.column_defaults) { - if (allow_alter && (data.getColumnsListNonMaterialized().sizeOfDifference(columns) <= 2 || skip_sanity_checks)) + if (allow_alter && + (skip_sanity_checks || + data.getColumnsListNonMaterialized().sizeOfDifference(columns) + + data.materialized_columns.sizeOfDifference(materialized_columns) <= 2)) { LOG_WARNING(log, "Table structure in ZooKeeper is a little different from local table structure. Assuming ALTER."); @@ -411,6 +425,9 @@ void StorageReplicatedMergeTree::createReplica() } zookeeper->create(replica_path + "/columns", data.getColumnsListNonMaterialized().toString(), zkutil::CreateMode::Persistent); + zookeeper->create(replica_path + "/materialized_columns", data.materialized_columns.toString(), zkutil::CreateMode::Persistent); + zookeeper->create(replica_path + "/alias_columns", data.alias_columns.toString(), zkutil::CreateMode::Persistent); + zookeeper->create(replica_path + "/column_defaults", data.column_defaults.toString(), zkutil::CreateMode::Persistent); } void StorageReplicatedMergeTree::activateReplica() @@ -632,6 +649,15 @@ void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper(const MergeTreeData: ops.push_back(new zkutil::Op::Check( zookeeper_path + "/columns", expected_columns_version)); + ops.push_back(new zkutil::Op::Check( + zookeeper_path + "/materialized_columns", + expected_columns_version)); + ops.push_back(new zkutil::Op::Check( + zookeeper_path + "/alias_columns", + expected_columns_version)); + ops.push_back(new zkutil::Op::Check( + zookeeper_path + "/column_defaults", + expected_columns_version)); ops.push_back(new zkutil::Op::Create( replica_path + "/parts/" + part_name, "", @@ -1588,10 +1614,15 @@ void StorageReplicatedMergeTree::alterThread() */ zkutil::Stat stat; - String columns_str = zookeeper->get(zookeeper_path + "/columns", &stat, alter_thread_event); + const String columns_str = zookeeper->get(zookeeper_path + "/columns", &stat, alter_thread_event); + const String materialized_columns_str = zookeeper->get(zookeeper_path + "/materialized_columns", + &stat, alter_thread_event); + const String alias_columns_str = zookeeper->get(zookeeper_path + "/alias_columns", + &stat, alter_thread_event); NamesAndTypesList columns = NamesAndTypesList::parse(columns_str, context.getDataTypeFactory()); - NamesAndTypesList materialized_columns; - NamesAndTypesList alias_columns; + NamesAndTypesList materialized_columns = NamesAndTypesList::parse( + materialized_columns_str, context.getDataTypeFactory()); + NamesAndTypesList alias_columns = NamesAndTypesList::parse(alias_columns_str, context.getDataTypeFactory()); ColumnDefaults column_defaults; bool changed_version = (stat.version != columns_version); @@ -1603,24 +1634,45 @@ void StorageReplicatedMergeTree::alterThread() { auto table_lock = lockStructureForAlter(); - if (columns != data.getColumnsListNonMaterialized()) + const auto columns_changed = columns != data.getColumnsListNonMaterialized(); + const auto materialized_columns_changed = materialized_columns != data.materialized_columns; + const auto alias_columns_changed = alias_columns != data.alias_columns; + const auto column_defaults_changed = false; + + if (columns_changed || materialized_columns_changed || alias_columns_changed || + column_defaults_changed) { LOG_INFO(log, "Columns list changed in ZooKeeper. Applying changes locally."); InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, materialized_columns, alias_columns, column_defaults, context); - this->materialized_columns = materialized_columns; - this->alias_columns = alias_columns; - this->column_defaults = column_defaults; - - data.setColumnsList(columns); - data.materialized_columns = std::move(materialized_columns); - data.alias_columns = std::move(alias_columns); - data.column_defaults = std::move(column_defaults); + if (columns_changed) + { + data.setColumnsList(columns); + + if (unreplicated_data) + unreplicated_data->setColumnsList(columns); + } + + if (materialized_columns_changed) + { + this->materialized_columns = materialized_columns; + data.materialized_columns = std::move(materialized_columns); + } + + if (alias_columns_changed) + { + this->alias_columns = alias_columns; + data.alias_columns = std::move(alias_columns); + } + + if (column_defaults_changed) + { + this->column_defaults = column_defaults; + data.column_defaults = std::move(column_defaults); + } - if (unreplicated_data) - unreplicated_data->setColumnsList(columns); LOG_INFO(log, "Applied changes to table."); } else @@ -1647,12 +1699,14 @@ void StorageReplicatedMergeTree::alterThread() if (!changed_version) parts = data.getDataParts(); + const auto columns_plus_materialized = data.getColumnsList(); + for (const MergeTreeData::DataPartPtr & part : parts) { /// Обновим кусок и запишем результат во временные файлы. /// TODO: Можно пропускать проверку на слишком большие изменения, если в ZooKeeper есть, например, /// нода /flags/force_alter. - auto transaction = data.alterDataPart(part, columns); + auto transaction = data.alterDataPart(part, columns_plus_materialized); if (!transaction) continue; @@ -1676,7 +1730,7 @@ void StorageReplicatedMergeTree::alterThread() for (const MergeTreeData::DataPartPtr & part : parts) { - auto transaction = unreplicated_data->alterDataPart(part, columns); + auto transaction = unreplicated_data->alterDataPart(part, columns_plus_materialized); if (!transaction) continue; @@ -1689,6 +1743,9 @@ void StorageReplicatedMergeTree::alterThread() /// Список столбцов для конкретной реплики. zookeeper->set(replica_path + "/columns", columns.toString()); + zookeeper->set(replica_path + "/materialized_columns", materialized_columns.toString()); + zookeeper->set(replica_path + "/alias_columns", alias_columns.toString()); + zookeeper->set(replica_path + "/column_defaults", column_defaults.toString()); if (changed_version) { @@ -2330,6 +2387,9 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, NamesAndTypesList new_alias_columns; ColumnDefaults new_column_defaults; String new_columns_str; + String new_materialized_columns_str; + String new_alias_columns_str; + String new_column_defaults_str; int new_columns_version; zkutil::Stat stat; @@ -2348,9 +2408,15 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); new_columns_str = new_columns.toString(); + new_materialized_columns_str = new_materialized_columns.toString(); + new_alias_columns_str = new_alias_columns.toString(); + new_column_defaults_str = new_column_defaults.toString(); /// Делаем ALTER. zookeeper->set(zookeeper_path + "/columns", new_columns_str, -1, &stat); + zookeeper->set(zookeeper_path + "/materialized_columns", new_materialized_columns_str, -1, &stat); + zookeeper->set(zookeeper_path + "/alias_columns", new_alias_columns_str, -1, &stat); + zookeeper->set(zookeeper_path + "/column_defaults", new_column_defaults_str, -1, &stat); new_columns_version = stat.version; } From 934149d59a41ba7848709228999513c025b87af9 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 23 Oct 2014 16:16:24 +0400 Subject: [PATCH 19/29] Merge --- dbms/include/DB/Storages/ColumnDefault.h | 67 ++++- .../Storages/StorageReplicatedMergeTree.cpp | 39 ++- ...cated_merge_tree_alter_zookeeper.reference | 284 +++++++++--------- 3 files changed, 241 insertions(+), 149 deletions(-) diff --git a/dbms/include/DB/Storages/ColumnDefault.h b/dbms/include/DB/Storages/ColumnDefault.h index 4da32b40dc3..f58bf75a11f 100644 --- a/dbms/include/DB/Storages/ColumnDefault.h +++ b/dbms/include/DB/Storages/ColumnDefault.h @@ -2,6 +2,9 @@ #include #include +#include +#include +#include #include namespace DB @@ -67,7 +70,67 @@ namespace DB using std::unordered_map::unordered_map; /// @todo implement (de)serialization - String toString() const { return {}; } - static ColumnDefaults parse(const String & str) { return {}; } + String toString() const + { + String s; + WriteBufferFromString buf{s}; + + writeString("column defaults format version: 1\n", buf); + writeText(size(), buf); + writeString(" columns:\n", buf); + + for (const auto & column_default : *this) + { + writeBackQuotedString(column_default.first, buf); + writeChar(' ', buf); + writeString(DB::toString(column_default.second.type), buf); + writeChar('\t', buf); + writeString(queryToString(column_default.second.expression), buf); + writeChar('\n', buf); + } + + return s; + } + + static ColumnDefaults parse(const String & str) { + ReadBufferFromString buf{str}; + ColumnDefaults defaults{}; + + assertString("column defaults format version: 1\n", buf); + size_t count{}; + readText(count, buf); + assertString(" columns:\n", buf); + + ParserTernaryOperatorExpression expr_parser; + + for (size_t i = 0; i < count; ++i) + { + String column_name; + readBackQuotedString(column_name, buf); + assertString(" ", buf); + + String default_type_str; + readString(default_type_str, buf); + const auto default_type = columnDefaultTypeFromString(default_type_str); + assertString("\t", buf); + + String default_expr_str; + readText(default_expr_str, buf); + assertString("\n", buf); + + ASTPtr default_expr; + Expected expected{}; + auto begin = default_expr_str.data(); + const auto end = begin + default_expr_str.size(); + if (!expr_parser.parse(begin, end, default_expr, expected)) + throw Exception{"Could not parse default expression", DB::ErrorCodes::CANNOT_PARSE_TEXT}; + + defaults.emplace(column_name, ColumnDefault{default_type, default_expr}); + } + + assertEOF(buf); + + return defaults; + } }; } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index aaea1e3cc96..9364996da10 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1619,11 +1619,13 @@ void StorageReplicatedMergeTree::alterThread() &stat, alter_thread_event); const String alias_columns_str = zookeeper->get(zookeeper_path + "/alias_columns", &stat, alter_thread_event); + const String column_defaults_str = zookeeper->get(zookeeper_path + "/column_defaults", + &stat, alter_thread_event); NamesAndTypesList columns = NamesAndTypesList::parse(columns_str, context.getDataTypeFactory()); NamesAndTypesList materialized_columns = NamesAndTypesList::parse( materialized_columns_str, context.getDataTypeFactory()); NamesAndTypesList alias_columns = NamesAndTypesList::parse(alias_columns_str, context.getDataTypeFactory()); - ColumnDefaults column_defaults; + ColumnDefaults column_defaults = ColumnDefaults::parse(column_defaults_str); bool changed_version = (stat.version != columns_version); @@ -1637,7 +1639,7 @@ void StorageReplicatedMergeTree::alterThread() const auto columns_changed = columns != data.getColumnsListNonMaterialized(); const auto materialized_columns_changed = materialized_columns != data.materialized_columns; const auto alias_columns_changed = alias_columns != data.alias_columns; - const auto column_defaults_changed = false; + const auto column_defaults_changed = column_defaults != data.column_defaults; if (columns_changed || materialized_columns_changed || alias_columns_changed || column_defaults_changed) @@ -2444,9 +2446,18 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, while (!shutdown_called) { String replica_columns_str; + String replica_materialized_columns_str; + String replica_alias_columns_str; + String replica_column_defaults_str; /// Реплику могли успеть удалить. - if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/columns", replica_columns_str, &stat)) + if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/columns", replica_columns_str, &stat) || + !zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/materialized_columns", + replica_materialized_columns_str, &stat) || + !zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/alias_columns", + replica_alias_columns_str, &stat) || + !zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/column_defaults", + replica_column_defaults_str, &stat)) { LOG_WARNING(log, replica << " was removed"); break; @@ -2454,12 +2465,24 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, int replica_columns_version = stat.version; - if (replica_columns_str == new_columns_str) + if (replica_columns_str == new_columns_str && + replica_materialized_columns_str == new_materialized_columns_str && + replica_alias_columns_str == new_alias_columns_str && + replica_column_defaults_str == new_column_defaults_str) break; if (!zookeeper->exists(zookeeper_path + "/columns", &stat)) throw Exception(zookeeper_path + "/columns doesn't exist", ErrorCodes::NOT_FOUND_NODE); + if (!zookeeper->exists(zookeeper_path + "/materialized_columns", &stat)) + throw Exception(zookeeper_path + "/materialized_columns doesn't exist", ErrorCodes::NOT_FOUND_NODE); + + if (!zookeeper->exists(zookeeper_path + "/alias_columns", &stat)) + throw Exception(zookeeper_path + "/alias_columns doesn't exist", ErrorCodes::NOT_FOUND_NODE); + + if (!zookeeper->exists(zookeeper_path + "/column_defaults", &stat)) + throw Exception(zookeeper_path + "/column_defaults doesn't exist", ErrorCodes::NOT_FOUND_NODE); + if (stat.version != new_columns_version) { LOG_WARNING(log, zookeeper_path + "/columns changed before ALTER finished; " @@ -2467,7 +2490,13 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, return; } - if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/columns", &stat, alter_query_event)) + if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/columns", &stat, alter_query_event) || + !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/materialized_columns", + &stat, alter_query_event) || + !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/alias_columns", + &stat, alter_query_event) || + !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/column_defaults", + &stat, alter_query_event)) { LOG_WARNING(log, replica << " was removed"); break; diff --git a/dbms/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference b/dbms/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference index 242ec717050..30ebde64349 100644 --- a/dbms/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference +++ b/dbms/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.reference @@ -1,122 +1,122 @@ -d Date -k UInt64 -i32 Int32 +d Date +k UInt64 +i32 Int32 CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 +d Date +k UInt64 +i32 Int32 CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 10 42 -d Date -k UInt64 -i32 Int32 -dt DateTime +d Date +k UInt64 +i32 Int32 +dt DateTime CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt DateTime) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt DateTime +d Date +k UInt64 +i32 Int32 +dt DateTime CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTime) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 9 41 1992-01-01 08:00:00 2015-01-01 10 42 0000-00-00 00:00:00 -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] 2015-01-01 10 42 0000-00-00 00:00:00 [] [] -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -n.d Array(Date) +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +n.d Array(Date) CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -n.d Array(Date) +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +n.d Array(Date) CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] [] 2015-01-01 10 42 0000-00-00 00:00:00 [] [] [] -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -n.d Array(Date) -s String +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +n.d Array(Date) +s String CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date), s String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -n.d Array(Date) -s String +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +n.d Array(Date) +s String CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date), s String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] ['2000-01-01','2000-01-01','2000-01-03'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] ['2000-01-01','2000-01-01','2000-01-03'] 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] [] 2015-01-01 10 42 0000-00-00 00:00:00 [] [] [] -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -s Int64 +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +s Int64 CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), s Int64) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -s Int64 +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +s Int64 CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), s Int64) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] 0 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 0 2015-01-01 9 41 1992-01-01 08:00:00 [] [] 0 2015-01-01 10 42 0000-00-00 00:00:00 [] [] 0 -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -s UInt32 -n.d Array(Date) +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +s UInt32 +n.d Array(Date) CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), s UInt32, `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt DateTime -n.ui8 Array(UInt8) -n.s Array(String) -s UInt32 -n.d Array(Date) +d Date +k UInt64 +i32 Int32 +dt DateTime +n.ui8 Array(UInt8) +n.s Array(String) +s UInt32 +n.d Array(Date) CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), s UInt32, `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 7 39 2014-07-14 13:26:50 [10,20,30] ['120','130','140'] 0 ['0000-00-00','0000-00-00','0000-00-00'] @@ -128,91 +128,91 @@ CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTim 2015-01-01 8 40 2012-12-12 12:12:12 [1,2,3] ['12','13','14'] 0 ['0000-00-00','0000-00-00','0000-00-00'] 2015-01-01 9 41 1992-01-01 08:00:00 [] [] 0 [] 2015-01-01 10 42 0000-00-00 00:00:00 [] [] 0 [] -d Date -k UInt64 -i32 Int32 -dt DateTime -n.s Array(String) -s UInt32 +d Date +k UInt64 +i32 Int32 +dt DateTime +n.s Array(String) +s UInt32 CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.s` Array(String), s UInt32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt DateTime -n.s Array(String) -s UInt32 +d Date +k UInt64 +i32 Int32 +dt DateTime +n.s Array(String) +s UInt32 CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTime, `n.s` Array(String), s UInt32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 ['asd','qwe','qwe'] 100500 2015-01-01 7 39 2014-07-14 13:26:50 ['120','130','140'] 0 2015-01-01 8 40 2012-12-12 12:12:12 ['12','13','14'] 0 2015-01-01 9 41 1992-01-01 08:00:00 [] 0 2015-01-01 10 42 0000-00-00 00:00:00 [] 0 -d Date -k UInt64 -i32 Int32 -dt DateTime -s UInt32 +d Date +k UInt64 +i32 Int32 +dt DateTime +s UInt32 CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt DateTime, s UInt32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt DateTime -s UInt32 +d Date +k UInt64 +i32 Int32 +dt DateTime +s UInt32 CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTime, s UInt32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 2015-01-01 7 39 2014-07-14 13:26:50 0 2015-01-01 8 40 2012-12-12 12:12:12 0 2015-01-01 9 41 1992-01-01 08:00:00 0 2015-01-01 10 42 0000-00-00 00:00:00 0 -d Date -k UInt64 -i32 Int32 -dt DateTime -s UInt32 -n.s Array(String) -n.d Array(Date) +d Date +k UInt64 +i32 Int32 +dt DateTime +s UInt32 +n.s Array(String) +n.d Array(Date) CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt DateTime, s UInt32, `n.s` Array(String), `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt DateTime -s UInt32 -n.s Array(String) -n.d Array(Date) +d Date +k UInt64 +i32 Int32 +dt DateTime +s UInt32 +n.s Array(String) +n.d Array(Date) CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTime, s UInt32, `n.s` Array(String), `n.d` Array(Date)) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 [] [] 2015-01-01 7 39 2014-07-14 13:26:50 0 [] [] 2015-01-01 8 40 2012-12-12 12:12:12 0 [] [] 2015-01-01 9 41 1992-01-01 08:00:00 0 [] [] 2015-01-01 10 42 0000-00-00 00:00:00 0 [] [] -d Date -k UInt64 -i32 Int32 -dt DateTime -s UInt32 +d Date +k UInt64 +i32 Int32 +dt DateTime +s UInt32 CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt DateTime, s UInt32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt DateTime -s UInt32 +d Date +k UInt64 +i32 Int32 +dt DateTime +s UInt32 CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt DateTime, s UInt32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 13:26:50 100500 2015-01-01 7 39 2014-07-14 13:26:50 0 2015-01-01 8 40 2012-12-12 12:12:12 0 2015-01-01 9 41 1992-01-01 08:00:00 0 2015-01-01 10 42 0000-00-00 00:00:00 0 -d Date -k UInt64 -i32 Int32 -dt Date -s DateTime +d Date +k UInt64 +i32 Int32 +dt Date +s DateTime CREATE TABLE test.replicated_alter1 ( d Date, k UInt64, i32 Int32, dt Date, s DateTime) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r1\', d, k, 8192) -d Date -k UInt64 -i32 Int32 -dt Date -s DateTime +d Date +k UInt64 +i32 Int32 +dt Date +s DateTime CREATE TABLE test.replicated_alter2 ( d Date, k UInt64, i32 Int32, dt Date, s DateTime) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test/alter\', \'r2\', d, k, 8192) 2015-01-01 6 38 2014-07-15 1970-01-02 06:55:00 2015-01-01 7 39 2014-07-14 0000-00-00 00:00:00 From ec5aa734f9a0b89c1065b39ffec79016c794060a Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 23 Oct 2014 17:53:16 +0400 Subject: [PATCH 20/29] dbms: cut coupling between dbms and dbms_functions. [#METR-12739] --- dbms/include/DB/Core/Block.h | 3 -- .../AddingDefaultBlockInputStream.h | 4 +- .../AddingDefaultBlockOutputStream.h | 4 +- .../DB/Interpreters/evaluateMissingDefaults.h | 49 +++++++++++++++++++ dbms/include/DB/Storages/ColumnDefault.h | 1 - .../DB/Storages/MergeTree/MergeTreeReader.h | 3 +- dbms/include/DB/Storages/StorageLog.h | 6 +++ dbms/src/Core/Block.cpp | 36 -------------- dbms/src/Storages/StorageLog.cpp | 13 +++++ 9 files changed, 76 insertions(+), 43 deletions(-) create mode 100644 dbms/include/DB/Interpreters/evaluateMissingDefaults.h diff --git a/dbms/include/DB/Core/Block.h b/dbms/include/DB/Core/Block.h index 05a8d5a47de..cf9e96d2d41 100644 --- a/dbms/include/DB/Core/Block.h +++ b/dbms/include/DB/Core/Block.h @@ -66,9 +66,6 @@ public: void erase(const String & name); /// Добавляет в блок недостающие столбцы со значениями по-умолчанию void addDefaults(const NamesAndTypesList & required_columns); - void addDefaults(const NamesAndTypesList & required_columns, - const ColumnDefaults & column_defaults, - const Context & context); ColumnWithNameAndType & getByPosition(size_t position); const ColumnWithNameAndType & getByPosition(size_t position) const; diff --git a/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h b/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h index 76b10b7d0e1..92928a4167a 100644 --- a/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h +++ b/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -54,7 +55,8 @@ protected: Block res = children.back()->read(); if (!res) return res; - res.addDefaults(*required_columns, column_defaults, context); + evaluateMissingDefaults(res, *required_columns, column_defaults, context); + res.addDefaults(*required_columns); return res; } diff --git a/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h b/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h index 3d2fd157f65..efdcb0f36fa 100644 --- a/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h +++ b/dbms/include/DB/DataStreams/AddingDefaultBlockOutputStream.h @@ -7,6 +7,7 @@ #include #include +#include namespace DB @@ -38,7 +39,8 @@ public: void write(const Block & block) override { Block res = block; - res.addDefaults(*required_columns, column_defaults, context); + evaluateMissingDefaults(res, *required_columns, column_defaults, context); + res.addDefaults(*required_columns); output->write(res); } diff --git a/dbms/include/DB/Interpreters/evaluateMissingDefaults.h b/dbms/include/DB/Interpreters/evaluateMissingDefaults.h new file mode 100644 index 00000000000..ec695da4dbe --- /dev/null +++ b/dbms/include/DB/Interpreters/evaluateMissingDefaults.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +inline void evaluateMissingDefaults(Block & block, + const NamesAndTypesList & required_columns, + const ColumnDefaults & column_defaults, + const Context & context) +{ + if (column_defaults.empty()) + return; + + ASTPtr default_expr_list{stdext::make_unique().release()}; + + for (const auto & column : required_columns) + { + if (block.has(column.name)) + continue; + + const auto it = column_defaults.find(column.name); + + /// expressions must be cloned to prevent modification by the ExpressionAnalyzer + if (it != column_defaults.end()) + default_expr_list->children.emplace_back( + setAlias(it->second.expression->clone(), it->first)); + } + + /// nothing to evaluate + if (default_expr_list->children.empty()) + return; + + /** ExpressionAnalyzer eliminates "unused" columns, in order to ensure their safety + * we are going to operate on a copy instead of the original block */ + Block copy_block{block}; + /// evaluate default values for defaulted columns + ExpressionAnalyzer{default_expr_list, context, required_columns}.getActions(true)->execute(copy_block); + + /// move evaluated columns to the original block + for (auto & column_name_type : copy_block.getColumns()) + block.insert(std::move(column_name_type)); +} + +} diff --git a/dbms/include/DB/Storages/ColumnDefault.h b/dbms/include/DB/Storages/ColumnDefault.h index f58bf75a11f..b6a0b7a763b 100644 --- a/dbms/include/DB/Storages/ColumnDefault.h +++ b/dbms/include/DB/Storages/ColumnDefault.h @@ -4,7 +4,6 @@ #include #include #include -#include #include namespace DB diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h b/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h index a72086c1f4d..7b7d62a9127 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB @@ -195,7 +196,7 @@ public: } /// evaluate defaulted columns - res.addDefaults(columns, storage.column_defaults, storage.context); + evaluateMissingDefaults(res, columns, storage.column_defaults, storage.context); } catch (const Exception & e) { diff --git a/dbms/include/DB/Storages/StorageLog.h b/dbms/include/DB/Storages/StorageLog.h index 824b3adff17..8cfef75522e 100644 --- a/dbms/include/DB/Storages/StorageLog.h +++ b/dbms/include/DB/Storages/StorageLog.h @@ -148,6 +148,12 @@ public: const ColumnDefaults & column_defaults_, size_t max_compress_block_size_ = DEFAULT_MAX_COMPRESS_BLOCK_SIZE); + static StoragePtr create( + const std::string & path_, + const std::string & name_, + NamesAndTypesListPtr columns_, + size_t max_compress_block_size_ = DEFAULT_MAX_COMPRESS_BLOCK_SIZE); + std::string getName() const { return "Log"; } std::string getTableName() const { return name; } diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index 47f3da4165e..ca46b28cd59 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -34,42 +34,6 @@ void Block::addDefaults(const NamesAndTypesList & required_columns) insertDefault(column.name, column.type); } -void Block::addDefaults(const NamesAndTypesList & required_columns, - const ColumnDefaults & column_defaults, - const Context & context) -{ - ASTPtr default_expr_list{stdext::make_unique().release()}; - - for (const auto & column : required_columns) - { - if (has(column.name)) - continue; - - const auto it = column_defaults.find(column.name); - - /// expressions must be cloned to prevent modification by the ExpressionAnalyzer - if (it == column_defaults.end()) - insertDefault(column.name, column.type); - else - default_expr_list->children.emplace_back( - setAlias(it->second.expression->clone(), it->first)); - } - - /// nothing to evaluate - if (default_expr_list->children.empty()) - return; - - /** ExpressionAnalyzer eliminates "unused" columns, in order to ensure their safety - * we are going to operate on a copy instead of the original block */ - Block copy_block{*this}; - /// evaluate default values for defaulted columns - ExpressionAnalyzer{default_expr_list, context, required_columns}.getActions(true)->execute(copy_block); - - /// move evaluated columns to the original block - for (auto & column_name_type : copy_block.getColumns()) - insert(std::move(column_name_type)); -} - Block & Block::operator= (const Block & other) { data = other.data; diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 049540736bb..9a1732ef16d 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -450,6 +450,19 @@ StoragePtr StorageLog::create( })->thisPtr(); } +StoragePtr StorageLog::create( + const std::string & path_, + const std::string & name_, + NamesAndTypesListPtr columns_, + size_t max_compress_block_size_) +{ + return (new StorageLog{ + path_, name_, columns_, + {}, {}, {}, + max_compress_block_size_ + })->thisPtr(); +} + void StorageLog::addFile(const String & column_name, const IDataType & type, size_t level) { From 2798a19944dce4f515d10d0f788bd0bd433fe83f Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Fri, 24 Oct 2014 14:48:47 +0400 Subject: [PATCH 21/29] Merge --- .../MergeTree/MergeTreeBlockInputStream.h | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h b/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h index ff8e7254cfe..a214ab2ba8e 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h @@ -99,6 +99,47 @@ protected: /// Будем вызывать progressImpl самостоятельно. void progress(size_t rows, size_t bytes) {} + void injectRequiredColumns(NamesAndTypesList & columns) const { + std::set required_columns; + auto modified = false; + + for (auto it = std::begin(columns); it != std::end(columns);) + { + required_columns.emplace(*it); + + if (!owned_data_part->hasColumnFiles(it->name)) + { + const auto default_it = storage.column_defaults.find(it->name); + if (default_it != std::end(storage.column_defaults)) + { + IdentifierNameSet identifiers; + default_it->second.expression->collectIdentifierNames(identifiers); + + for (const auto & identifier : identifiers) + { + if (storage.hasColumn(identifier)) + { + NameAndTypePair column{identifier, storage.getDataTypeByName(identifier)}; + if (required_columns.count(column) == 0) + { + it = columns.emplace(++it, std::move(column)); + modified = true; + } + } + } + + if (modified) + continue; + } + } + + ++it; + } + + if (modified) + columns = NamesAndTypesList{std::begin(required_columns), std::end(required_columns)}; + } + Block readImpl() { Block res; @@ -108,6 +149,8 @@ protected: if (!reader) { + injectRequiredColumns(columns); + injectRequiredColumns(pre_columns); /// @todo resolve missing columns UncompressedCache * uncompressed_cache = use_uncompressed_cache ? storage.context.getUncompressedCache() : NULL; reader.reset(new MergeTreeReader(path, owned_data_part->name, columns, uncompressed_cache, storage, all_mark_ranges)); From 3407600536851c62f642d784537d060147f192d7 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Mon, 10 Nov 2014 19:16:43 +0300 Subject: [PATCH 22/29] dbms: save all columns in a single ZooKeeper node [#METR-12739] --- dbms/include/DB/Core/Block.h | 1 - dbms/include/DB/Storages/ColumnDefault.h | 83 +++++++++---- .../Storages/StorageReplicatedMergeTree.cpp | 112 +++++------------- 3 files changed, 94 insertions(+), 102 deletions(-) diff --git a/dbms/include/DB/Core/Block.h b/dbms/include/DB/Core/Block.h index cf9e96d2d41..df308561726 100644 --- a/dbms/include/DB/Core/Block.h +++ b/dbms/include/DB/Core/Block.h @@ -21,7 +21,6 @@ namespace DB */ class Context; -struct ColumnDefaults; class Block { diff --git a/dbms/include/DB/Storages/ColumnDefault.h b/dbms/include/DB/Storages/ColumnDefault.h index b6a0b7a763b..954a8e4b81e 100644 --- a/dbms/include/DB/Storages/ColumnDefault.h +++ b/dbms/include/DB/Storages/ColumnDefault.h @@ -64,50 +64,86 @@ namespace DB return lhs.type == rhs.type && queryToString(lhs.expression) == queryToString(rhs.expression); } - struct ColumnDefaults : public std::unordered_map - { - using std::unordered_map::unordered_map; + using ColumnDefaults = std::unordered_map; + + template + struct ColumnsDescription + { + template using by_value_or_cref = typename std::conditional::type; + by_value_or_cref columns; + by_value_or_cref materialized; + by_value_or_cref alias; + by_value_or_cref defaults; - /// @todo implement (de)serialization String toString() const { String s; WriteBufferFromString buf{s}; - writeString("column defaults format version: 1\n", buf); - writeText(size(), buf); + writeString("columns format version: 1\n", buf); + writeText(columns.size() + materialized.size() + alias.size(), buf); writeString(" columns:\n", buf); - for (const auto & column_default : *this) - { - writeBackQuotedString(column_default.first, buf); - writeChar(' ', buf); - writeString(DB::toString(column_default.second.type), buf); - writeChar('\t', buf); - writeString(queryToString(column_default.second.expression), buf); - writeChar('\n', buf); - } + const auto write_columns = [this, &buf] (const NamesAndTypesList & columns) { + for (const auto & column : columns) + { + const auto it = defaults.find(column.name); + + writeBackQuotedString(column.name, buf); + writeChar(' ', buf); + writeString(column.type->getName(), buf); + if (it == std::end(defaults)) + { + writeChar('\n', buf); + continue; + } + else + writeChar('\t', buf); + + writeString(DB::toString(it->second.type), buf); + writeChar('\t', buf); + writeString(queryToString(it->second.expression), buf); + writeChar('\n', buf); + } + }; + + write_columns(columns); + write_columns(materialized); + write_columns(alias); return s; } - static ColumnDefaults parse(const String & str) { + static ColumnsDescription parse(const String & str, const DataTypeFactory & data_type_factory) + { ReadBufferFromString buf{str}; - ColumnDefaults defaults{}; - assertString("column defaults format version: 1\n", buf); + assertString("columns format version: 1\n", buf); size_t count{}; readText(count, buf); assertString(" columns:\n", buf); ParserTernaryOperatorExpression expr_parser; + ColumnsDescription result{}; for (size_t i = 0; i < count; ++i) { String column_name; readBackQuotedString(column_name, buf); assertString(" ", buf); + String type_name; + readString(type_name, buf); + auto type = data_type_factory.get(type_name); + if (*buf.position() == '\n') + { + assertString("\n", buf); + + result.columns.emplace_back(column_name, std::move(type)); + continue; + } + assertString("\t", buf); + String default_type_str; readString(default_type_str, buf); const auto default_type = columnDefaultTypeFromString(default_type_str); @@ -124,12 +160,19 @@ namespace DB if (!expr_parser.parse(begin, end, default_expr, expected)) throw Exception{"Could not parse default expression", DB::ErrorCodes::CANNOT_PARSE_TEXT}; - defaults.emplace(column_name, ColumnDefault{default_type, default_expr}); + if (ColumnDefaultType::Default == default_type) + result.columns.emplace_back(column_name, std::move(type)); + else if (ColumnDefaultType::Materialized == default_type) + result.materialized.emplace_back(column_name, std::move(type)); + else if (ColumnDefaultType::Alias == default_type) + result.alias.emplace_back(column_name, std::move(type)); + + result.defaults.emplace(column_name, ColumnDefault{default_type, default_expr}); } assertEOF(buf); - return defaults; + return result; } }; } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 9364996da10..9d8eeb2c022 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -207,13 +207,9 @@ void StorageReplicatedMergeTree::createTableIfNotExists() zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); ops.push_back(new zkutil::Op::Create(zookeeper_path + "/metadata", metadata.str(), zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); - ops.push_back(new zkutil::Op::Create(zookeeper_path + "/columns", data.getColumnsListNonMaterialized().toString(), - zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); - ops.push_back(new zkutil::Op::Create(zookeeper_path + "/materialized_columns", data.materialized_columns.toString(), - zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); - ops.push_back(new zkutil::Op::Create(zookeeper_path + "/alias_columns", data.alias_columns.toString(), - zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); - ops.push_back(new zkutil::Op::Create(zookeeper_path + "/column_defaults", data.column_defaults.toString(), + ops.push_back(new zkutil::Op::Create(zookeeper_path + "/columns", ColumnsDescription{ + data.getColumnsListNonMaterialized(), data.materialized_columns, + data.alias_columns, data.column_defaults}.toString(), zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); ops.push_back(new zkutil::Op::Create(zookeeper_path + "/log", "", zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); @@ -261,13 +257,15 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo assertEOF(buf); zkutil::Stat stat; - auto columns = NamesAndTypesList::parse(zookeeper->get(zookeeper_path + "/columns", &stat), context.getDataTypeFactory()); - auto materialized_columns = NamesAndTypesList::parse( - zookeeper->get(zookeeper_path + "/materialized_columns", &stat), context.getDataTypeFactory()); - auto alias_columns = NamesAndTypesList::parse( - zookeeper->get(zookeeper_path + "/alias_columns", &stat), context.getDataTypeFactory()); - auto column_defaults = ColumnDefaults::parse(zookeeper->get(zookeeper_path + "/column_defaults", &stat)); + auto columns_desc = ColumnsDescription::parse( + zookeeper->get(zookeeper_path + "/columns", &stat), context.getDataTypeFactory()); + + auto & columns = columns_desc.columns; + auto & materialized_columns = columns_desc.materialized; + auto & alias_columns = columns_desc.alias; + auto & column_defaults = columns_desc.defaults; columns_version = stat.version; + if (columns != data.getColumnsListNonMaterialized() || materialized_columns != data.materialized_columns || alias_columns != data.alias_columns || @@ -424,10 +422,12 @@ void StorageReplicatedMergeTree::createReplica() LOG_DEBUG(log, "Copied " << source_queue.size() << " queue entries"); } - zookeeper->create(replica_path + "/columns", data.getColumnsListNonMaterialized().toString(), zkutil::CreateMode::Persistent); - zookeeper->create(replica_path + "/materialized_columns", data.materialized_columns.toString(), zkutil::CreateMode::Persistent); - zookeeper->create(replica_path + "/alias_columns", data.alias_columns.toString(), zkutil::CreateMode::Persistent); - zookeeper->create(replica_path + "/column_defaults", data.column_defaults.toString(), zkutil::CreateMode::Persistent); + zookeeper->create(replica_path + "/columns", ColumnsDescription{ + data.getColumnsListNonMaterialized(), + data.materialized_columns, + data.alias_columns, + data.column_defaults + }.toString(), zkutil::CreateMode::Persistent); } void StorageReplicatedMergeTree::activateReplica() @@ -649,15 +649,6 @@ void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper(const MergeTreeData: ops.push_back(new zkutil::Op::Check( zookeeper_path + "/columns", expected_columns_version)); - ops.push_back(new zkutil::Op::Check( - zookeeper_path + "/materialized_columns", - expected_columns_version)); - ops.push_back(new zkutil::Op::Check( - zookeeper_path + "/alias_columns", - expected_columns_version)); - ops.push_back(new zkutil::Op::Check( - zookeeper_path + "/column_defaults", - expected_columns_version)); ops.push_back(new zkutil::Op::Create( replica_path + "/parts/" + part_name, "", @@ -1615,17 +1606,12 @@ void StorageReplicatedMergeTree::alterThread() zkutil::Stat stat; const String columns_str = zookeeper->get(zookeeper_path + "/columns", &stat, alter_thread_event); - const String materialized_columns_str = zookeeper->get(zookeeper_path + "/materialized_columns", - &stat, alter_thread_event); - const String alias_columns_str = zookeeper->get(zookeeper_path + "/alias_columns", - &stat, alter_thread_event); - const String column_defaults_str = zookeeper->get(zookeeper_path + "/column_defaults", - &stat, alter_thread_event); - NamesAndTypesList columns = NamesAndTypesList::parse(columns_str, context.getDataTypeFactory()); - NamesAndTypesList materialized_columns = NamesAndTypesList::parse( - materialized_columns_str, context.getDataTypeFactory()); - NamesAndTypesList alias_columns = NamesAndTypesList::parse(alias_columns_str, context.getDataTypeFactory()); - ColumnDefaults column_defaults = ColumnDefaults::parse(column_defaults_str); + auto columns_desc = ColumnsDescription::parse(columns_str, context.getDataTypeFactory()); + + auto & columns = columns_desc.columns; + auto & materialized_columns = columns_desc.materialized; + auto & alias_columns = columns_desc.alias; + auto & column_defaults = columns_desc.defaults; bool changed_version = (stat.version != columns_version); @@ -1744,10 +1730,7 @@ void StorageReplicatedMergeTree::alterThread() } /// Список столбцов для конкретной реплики. - zookeeper->set(replica_path + "/columns", columns.toString()); - zookeeper->set(replica_path + "/materialized_columns", materialized_columns.toString()); - zookeeper->set(replica_path + "/alias_columns", alias_columns.toString()); - zookeeper->set(replica_path + "/column_defaults", column_defaults.toString()); + zookeeper->set(replica_path + "/columns", columns_str); if (changed_version) { @@ -2389,9 +2372,6 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, NamesAndTypesList new_alias_columns; ColumnDefaults new_column_defaults; String new_columns_str; - String new_materialized_columns_str; - String new_alias_columns_str; - String new_column_defaults_str; int new_columns_version; zkutil::Stat stat; @@ -2409,16 +2389,13 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, new_column_defaults = data.column_defaults; params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); - new_columns_str = new_columns.toString(); - new_materialized_columns_str = new_materialized_columns.toString(); - new_alias_columns_str = new_alias_columns.toString(); - new_column_defaults_str = new_column_defaults.toString(); + new_columns_str = ColumnsDescription{ + new_columns, new_materialized_columns, + new_alias_columns, new_column_defaults + }.toString(); /// Делаем ALTER. zookeeper->set(zookeeper_path + "/columns", new_columns_str, -1, &stat); - zookeeper->set(zookeeper_path + "/materialized_columns", new_materialized_columns_str, -1, &stat); - zookeeper->set(zookeeper_path + "/alias_columns", new_alias_columns_str, -1, &stat); - zookeeper->set(zookeeper_path + "/column_defaults", new_column_defaults_str, -1, &stat); new_columns_version = stat.version; } @@ -2446,18 +2423,9 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, while (!shutdown_called) { String replica_columns_str; - String replica_materialized_columns_str; - String replica_alias_columns_str; - String replica_column_defaults_str; /// Реплику могли успеть удалить. - if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/columns", replica_columns_str, &stat) || - !zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/materialized_columns", - replica_materialized_columns_str, &stat) || - !zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/alias_columns", - replica_alias_columns_str, &stat) || - !zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/column_defaults", - replica_column_defaults_str, &stat)) + if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/columns", replica_columns_str, &stat)) { LOG_WARNING(log, replica << " was removed"); break; @@ -2465,24 +2433,12 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, int replica_columns_version = stat.version; - if (replica_columns_str == new_columns_str && - replica_materialized_columns_str == new_materialized_columns_str && - replica_alias_columns_str == new_alias_columns_str && - replica_column_defaults_str == new_column_defaults_str) + if (replica_columns_str == new_columns_str) break; if (!zookeeper->exists(zookeeper_path + "/columns", &stat)) throw Exception(zookeeper_path + "/columns doesn't exist", ErrorCodes::NOT_FOUND_NODE); - if (!zookeeper->exists(zookeeper_path + "/materialized_columns", &stat)) - throw Exception(zookeeper_path + "/materialized_columns doesn't exist", ErrorCodes::NOT_FOUND_NODE); - - if (!zookeeper->exists(zookeeper_path + "/alias_columns", &stat)) - throw Exception(zookeeper_path + "/alias_columns doesn't exist", ErrorCodes::NOT_FOUND_NODE); - - if (!zookeeper->exists(zookeeper_path + "/column_defaults", &stat)) - throw Exception(zookeeper_path + "/column_defaults doesn't exist", ErrorCodes::NOT_FOUND_NODE); - if (stat.version != new_columns_version) { LOG_WARNING(log, zookeeper_path + "/columns changed before ALTER finished; " @@ -2490,13 +2446,7 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, return; } - if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/columns", &stat, alter_query_event) || - !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/materialized_columns", - &stat, alter_query_event) || - !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/alias_columns", - &stat, alter_query_event) || - !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/column_defaults", - &stat, alter_query_event)) + if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/columns", &stat, alter_query_event)) { LOG_WARNING(log, replica << " was removed"); break; From de271b753e5eee6c14ae612286f083e50dc434e4 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 11 Nov 2014 17:11:49 +0300 Subject: [PATCH 23/29] dbms: allow creating tables with all columns set to default. [#METR-12739] --- dbms/src/Interpreters/InterpreterAlterQuery.cpp | 1 - dbms/src/Interpreters/InterpreterCreateQuery.cpp | 11 ++++++----- dbms/src/Storages/AlterCommands.cpp | 5 +++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index b86da11bfdb..377b5ee50ed 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 5d5e3f31c50..fef8fcb043f 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -253,7 +254,7 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( auto & column_list_ast = typeid_cast(*expression_list); /// list of table columns in correct order - NamesAndTypesList columns{}, known_type_columns{}; + NamesAndTypesList columns{}; ColumnDefaults defaults{}; /// Columns requiring type-deduction or default_expression type-check @@ -273,10 +274,10 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( const auto & type_range = col_decl.type->range; columns.emplace_back(col_decl.name, context.getDataTypeFactory().get({ type_range.first, type_range.second })); - known_type_columns.emplace_back(columns.back()); } else - columns.emplace_back(col_decl.name, nullptr); + /// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions + columns.emplace_back(col_decl.name, new DataTypeUInt8); /// add column to postprocessing if there is a default_expression specified if (col_decl.default_expression) @@ -308,7 +309,7 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( /// set missing types and wrap default_expression's in a conversion-function if necessary if (!defaulted_columns.empty()) { - const auto actions = ExpressionAnalyzer{default_expr_list, context, known_type_columns}.getActions(true); + const auto actions = ExpressionAnalyzer{default_expr_list, context, columns}.getActions(true); const auto block = actions->getSampleBlock(); for (auto & column : defaulted_columns) @@ -316,7 +317,7 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( const auto name_and_type_ptr = column.first; const auto col_decl_ptr = column.second; - if (name_and_type_ptr->type) + if (col_decl_ptr->type) { const auto & tmp_column = block.getByName(col_decl_ptr->name + "_tmp"); diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 4bd280fdfbf..1e7ed02b581 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -184,8 +185,8 @@ namespace DB defaults.erase(command.column_name); } - if (command.data_type) - columns.emplace_back(command.column_name, command.data_type); + /// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions + columns.emplace_back(command.column_name, command.data_type ? command.data_type : new DataTypeUInt8); if (command.default_expression) { From b682e42e5a5c55a54479c513a5e1d9452808d498 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 11 Nov 2014 17:20:22 +0300 Subject: [PATCH 24/29] dbms: prohibit creating tables without physical columns. [#METR-12739] --- dbms/src/Interpreters/InterpreterCreateQuery.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index fef8fcb043f..bd68e0ca4fb 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -126,6 +126,9 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) alias_columns = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Alias); columns = new NamesAndTypesList{std::move(columns_and_defaults.first)}; column_defaults = std::move(columns_and_defaults.second); + + if (columns->size() + materialized_columns.size() == 0) + throw Exception{"Cannot CREATE table without physical columns", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; } else if (!create.as_table.empty()) { From 8a2de38dc7e63a9788666c0fb473e009870da5df Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 11 Nov 2014 18:22:39 +0300 Subject: [PATCH 25/29] dbms: materialize constants during merge, allow alteration between DEFAULT and MATERIALIZED. [#METR-12739] --- dbms/src/Storages/AlterCommands.cpp | 25 +++++++++++++++++-- .../MergeTree/MergeTreeDataMerger.cpp | 4 ++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 1e7ed02b581..d647e216919 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -99,9 +99,14 @@ namespace DB const auto had_default_expr = it != column_defaults.end(); const auto old_default_type = had_default_expr ? it->second.type : ColumnDefaultType{}; - if (old_default_type != default_type) + /// allow conversion between DEFAULT and MATERIALIZED + const auto default_materialized_conversion = + (old_default_type == ColumnDefaultType::Default && default_type == ColumnDefaultType::Materialized) || + (old_default_type == ColumnDefaultType::Materialized && default_type == ColumnDefaultType::Default); + + if (old_default_type != default_type && !default_materialized_conversion) throw Exception{"Cannot change column default specifier from " + toString(old_default_type) + - " to " + toString(default_type), 0 }; + " to " + toString(default_type), ErrorCodes::INCORRECT_QUERY}; /// find column or throw exception const auto find_column = [this] (NamesAndTypesList & columns) { @@ -114,6 +119,22 @@ namespace DB return it; }; + /// remove from the old list, add to the new list in case of DEFAULT <-> MATERIALIZED alteration + if (default_materialized_conversion) + { + const auto was_default = old_default_type == ColumnDefaultType::Default; + auto & old_columns = was_default ? columns : materialized_columns; + auto & new_columns = was_default ? materialized_columns : columns; + + const auto column_it = find_column(old_columns); + new_columns.emplace_back(*column_it); + old_columns.erase(column_it); + + /// do not forget to change the default type of old column + if (had_default_expr) + column_defaults[column_name].type = default_type; + } + /// find column in one of three column lists const auto column_it = find_column( default_type == ColumnDefaultType::Default ? columns : diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index 5c305c85b7f..71be19ac22b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -356,7 +357,8 @@ MergeTreeData::DataPartPtr MergeTreeDataMerger::mergeParts( __sync_add_and_fetch(&merge_entry->bytes_read_uncompressed, bytes); }); - src_streams.push_back(new ExpressionBlockInputStream(input.release(), data.getPrimaryExpression())); + src_streams.push_back(new MaterializingBlockInputStream{ + new ExpressionBlockInputStream(input.release(), data.getPrimaryExpression())}); sum_rows_approx += parts[i]->size * data.index_granularity; } From 65629ef560a1eb50ee87b551f5570c1c24b25439 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 11 Nov 2014 19:29:21 +0300 Subject: [PATCH 26/29] Merge --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 4cab03b3eb9..cf0bf3edd20 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -45,23 +45,27 @@ MergeTreeData::MergeTreeData( log_name(log_name_), log(&Logger::get(log_name + " (Data)")) { /// Проверяем, что столбец с датой существует и имеет тип Date. - { - auto it = columns->begin(); - for (; it != columns->end(); ++it) + const auto check_date_exists = [this] (const NamesAndTypesList & columns) { + for (const auto & column : columns) { - if (it->name == date_column_name) + if (column.name == date_column_name) { - if (!typeid_cast(&*it->type)) + if (!typeid_cast(column.type.get())) throw Exception("Date column (" + date_column_name + ") for storage of MergeTree family must have type Date." - " Provided column of type " + it->type->getName() + "." - " You may have separate column with type " + it->type->getName() + ".", ErrorCodes::BAD_TYPE_OF_FIELD); - break; + " Provided column of type " + column.type->getName() + "." + " You may have separate column with type " + column.type->getName() + ".", ErrorCodes::BAD_TYPE_OF_FIELD); + return true; } } - if (it == columns->end()) - throw Exception("Date column (" + date_column_name + ") does not exist in table declaration.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); - } + return false; + }; + + if (!check_date_exists(*columns) && !check_date_exists(materialized_columns)) + throw Exception{ + "Date column (" + date_column_name + ") does not exist in table declaration.", + ErrorCodes::NO_SUCH_COLUMN_IN_TABLE + }; /// создаём директорию, если её нет Poco::File(full_path).createDirectories(); @@ -75,9 +79,9 @@ MergeTreeData::MergeTreeData( sort_descr.push_back(SortColumnDescription(name, 1)); } - primary_expr = ExpressionAnalyzer(primary_expr_ast, context, *columns).getActions(false); + primary_expr = ExpressionAnalyzer(primary_expr_ast, context, getColumnsList()).getActions(false); - ExpressionActionsPtr projected_expr = ExpressionAnalyzer(primary_expr_ast, context, *columns).getActions(true); + ExpressionActionsPtr projected_expr = ExpressionAnalyzer(primary_expr_ast, context, getColumnsList()).getActions(true); primary_key_sample = projected_expr->getSampleBlock(); } From ccc770e5abc0e67aa7321e71f0579c97be02dd6c Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 11 Nov 2014 19:30:17 +0300 Subject: [PATCH 27/29] dbms: add tests for defaulted columns. [#METR-12739] --- .../00072_defaulted_columns.reference | 41 +++++++++++++++++++ .../0_stateless/00072_defaulted_columns.sql | 36 ++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00072_defaulted_columns.reference create mode 100644 dbms/tests/queries/0_stateless/00072_defaulted_columns.sql diff --git a/dbms/tests/queries/0_stateless/00072_defaulted_columns.reference b/dbms/tests/queries/0_stateless/00072_defaulted_columns.reference new file mode 100644 index 00000000000..36893cb3f76 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00072_defaulted_columns.reference @@ -0,0 +1,41 @@ +col1 UInt8 DEFAULT 0 +col1 UInt32 +col2 UInt64 DEFAULT col1 + 1 +col3 UInt64 MATERIALIZED col1 + 2 +col4 UInt64 ALIAS col1 + 3 +10 11 +12 13 +payload String +date Date MATERIALIZED today() +key UInt64 MATERIALIZED 0 * rand() +hello clickhouse +payload String +date Date MATERIALIZED today() +key UInt64 MATERIALIZED 0 * rand() +payload_length UInt64 MATERIALIZED length(payload) +hello clickhouse 16 +some string 11 +hello clickhouse 16 +hello clickhouse 16 +some string 11 +payload String +payload_length UInt64 DEFAULT length(payload) +date Date MATERIALIZED today() +key UInt64 MATERIALIZED 0 * rand() +hello clickhouse 16 +some string 11 +payload String +payload_length UInt16 DEFAULT length(payload) % 65535 +date Date MATERIALIZED today() +key UInt64 MATERIALIZED 0 * rand() +hello clickhouse 16 +some string 11 +payload String +payload_length UInt16 DEFAULT toUInt16(length(payload)) +date Date MATERIALIZED today() +key UInt64 MATERIALIZED 0 * rand() +payload String +date Date MATERIALIZED today() +key UInt64 MATERIALIZED 0 * rand() +hello clickhouse +some string diff --git a/dbms/tests/queries/0_stateless/00072_defaulted_columns.sql b/dbms/tests/queries/0_stateless/00072_defaulted_columns.sql new file mode 100644 index 00000000000..d27c4b50d31 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00072_defaulted_columns.sql @@ -0,0 +1,36 @@ +drop table if exists defaulted_test; + +create table defaulted_test (col1 default 0) engine=Memory; +desc table defaulted_test; +drop table defaulted_test; + +create table defaulted_test (col1 UInt32, col2 default col1 + 1, col3 materialized col1 + 2, col4 alias col1 + 3) engine=Memory; +desc table defaulted_test; +insert into defaulted_test (col1) values (10); +select * from defaulted_test; +select col3, col4 from defaulted_test; +drop table defaulted_test; + +create table defaulted_test (payload String, date materialized today(), key materialized 0 * rand()) engine=MergeTree(date, key, 8192); +desc table defaulted_test; +insert into defaulted_test (payload) values ('hello clickhouse'); +select * from defaulted_test; +alter table defaulted_test add column payload_length materialized length(payload); +desc table defaulted_test; +select *, payload_length from defaulted_test; +insert into defaulted_test (payload) values ('some string'); +select *, payload_length from defaulted_test; +optimize table defaulted_test; +select *, payload_length from defaulted_test; +alter table defaulted_test modify column payload_length default length(payload); +desc table defaulted_test; +select * from defaulted_test; +alter table defaulted_test modify column payload_length default length(payload) % 65535; +desc table defaulted_test; +select * from defaulted_test; +alter table defaulted_test modify column payload_length UInt16 default length(payload); +desc table defaulted_test; +alter table defaulted_test drop column payload_length; +desc table defaulted_test; +select * from defaulted_test; +drop table defaulted_test; From 85e511fdab7c2188e3f72757129981a628d45121 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Wed, 12 Nov 2014 20:23:26 +0300 Subject: [PATCH 28/29] dbms: refactor IFunction descendants' registration in FunctionFactory --- dbms/include/DB/Functions/FunctionFactory.h | 5 +- .../DB/Functions/FunctionsArithmetic.h | 38 +-- dbms/include/DB/Functions/FunctionsArray.h | 71 +++-- dbms/include/DB/Functions/FunctionsCoding.h | 40 ++- .../DB/Functions/FunctionsComparison.h | 19 +- .../DB/Functions/FunctionsConditional.h | 6 +- .../DB/Functions/FunctionsConversion.h | 50 ++-- dbms/include/DB/Functions/FunctionsDateTime.h | 74 ++++-- .../DB/Functions/FunctionsDictionaries.h | 243 +++++++++++++----- .../DB/Functions/FunctionsFormatting.h | 5 +- dbms/include/DB/Functions/FunctionsHashing.h | 36 ++- .../DB/Functions/FunctionsHigherOrder.h | 21 +- dbms/include/DB/Functions/FunctionsLogical.h | 21 +- .../DB/Functions/FunctionsMiscellaneous.h | 91 +++++-- dbms/include/DB/Functions/FunctionsRandom.h | 9 +- .../DB/Functions/FunctionsReinterpret.h | 36 +-- dbms/include/DB/Functions/FunctionsRound.h | 36 +-- dbms/include/DB/Functions/FunctionsString.h | 46 ++-- .../DB/Functions/FunctionsStringArray.h | 17 +- .../DB/Functions/FunctionsStringSearch.h | 35 ++- dbms/include/DB/Functions/FunctionsURL.h | 49 ++-- .../DB/Functions/FunctionsVisitParam.h | 80 +++--- dbms/src/Functions/FunctionsArithmetic.cpp | 30 +-- dbms/src/Functions/FunctionsArray.cpp | 45 ++-- dbms/src/Functions/FunctionsCoding.cpp | 20 +- dbms/src/Functions/FunctionsComparison.cpp | 16 +- dbms/src/Functions/FunctionsConditional.cpp | 6 +- dbms/src/Functions/FunctionsConversion.cpp | 32 +-- dbms/src/Functions/FunctionsDateTime.cpp | 56 ++-- dbms/src/Functions/FunctionsDictionaries.cpp | 58 ++--- dbms/src/Functions/FunctionsFormatting.cpp | 6 +- dbms/src/Functions/FunctionsHashing.cpp | 24 +- dbms/src/Functions/FunctionsHigherOrder.cpp | 20 +- dbms/src/Functions/FunctionsLogical.cpp | 12 +- dbms/src/Functions/FunctionsMiscellaneous.cpp | 36 ++- dbms/src/Functions/FunctionsRandom.cpp | 8 +- dbms/src/Functions/FunctionsReinterpret.cpp | 30 +-- dbms/src/Functions/FunctionsRound.cpp | 10 +- dbms/src/Functions/FunctionsString.cpp | 30 +-- dbms/src/Functions/FunctionsStringArray.cpp | 12 +- dbms/src/Functions/FunctionsStringSearch.cpp | 24 +- dbms/src/Functions/FunctionsURL.cpp | 44 ++-- dbms/src/Functions/FunctionsVisitParam.cpp | 18 +- 43 files changed, 893 insertions(+), 672 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionFactory.h b/dbms/include/DB/Functions/FunctionFactory.h index 5900365995a..71a7ea796ce 100644 --- a/dbms/include/DB/Functions/FunctionFactory.h +++ b/dbms/include/DB/Functions/FunctionFactory.h @@ -25,9 +25,10 @@ public: FunctionPtr get(const String & name, const Context & context) const; - void registerFunction(const String & name, Creator creator) + template void registerFunction() { - functions[name] = creator; + static_assert(std::is_same::value, "F::create has incorrect type"); + functions[F::name] = &F::create; } }; diff --git a/dbms/include/DB/Functions/FunctionsArithmetic.h b/dbms/include/DB/Functions/FunctionsArithmetic.h index 0c4542b43e4..72dcf215602 100644 --- a/dbms/include/DB/Functions/FunctionsArithmetic.h +++ b/dbms/include/DB/Functions/FunctionsArithmetic.h @@ -367,6 +367,10 @@ struct BinaryOperationTraits template