diff --git a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h index 9da89aaccb7..288f577b34d 100644 --- a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h @@ -21,7 +21,13 @@ public: /** Изменяет список столбцов в метаданных таблицы на диске. Нужно вызывать под TableStructureLock соответствующей таблицы. */ - static void updateMetadata(const String & database, const String & table, const NamesAndTypesList & columns, Context & context); + static void updateMetadata(const String & database, + const String & table, + const NamesAndTypesList & columns, + const NamesAndTypesList & materialized_columns, + const NamesAndTypesList & alias_columns, + const ColumnDefaults & column_defaults, + Context & context); private: struct PartitionCommand { @@ -58,7 +64,6 @@ private: static void parseAlter(const ASTAlterQuery::ParameterContainer & params, const DataTypeFactory & data_type_factory, AlterCommands & out_alter_commands, PartitionCommands & out_partition_commands); - - static void validateColumnChanges(ASTAlterQuery::ParameterContainer & params, const StoragePtr & table, const Context & context); }; + } diff --git a/dbms/include/DB/Storages/AlterCommands.h b/dbms/include/DB/Storages/AlterCommands.h index e122a9f801d..8f67c021e53 100644 --- a/dbms/include/DB/Storages/AlterCommands.h +++ b/dbms/include/DB/Storages/AlterCommands.h @@ -1,7 +1,7 @@ #pragma once + #include -#include -#include +#include namespace DB { @@ -23,6 +23,9 @@ struct AlterCommand /// Для ADD и MODIFY - новый тип столбца. DataTypePtr data_type; + ColumnDefaultType default_type{}; + ASTPtr default_expression{}; + /// Для ADD - после какого столбца добавить новый. Если пустая строка, добавить в конец. Добавить в начало сейчас нельзя. String after_column; @@ -34,88 +37,19 @@ struct AlterCommand return (name_with_dot == name_type.name.substr(0, name_without_dot.length() + 1) || name_without_dot == name_type.name); } - void apply(NamesAndTypesList & columns) const - { - if (type == ADD) - { - if (std::count_if(columns.begin(), columns.end(), std::bind(namesEqual, std::cref(column_name), std::placeholders::_1))) - throw Exception("Cannot add column " + column_name + ": column with this name already exisits.", - DB::ErrorCodes::ILLEGAL_COLUMN); - - if (DataTypeNested::extractNestedTableName(column_name) != column_name && - !typeid_cast(&*data_type)) - throw Exception("Can't add nested column " + column_name + " of non-array type " + data_type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - NamesAndTypesList::iterator insert_it = columns.end(); - if (!after_column.empty()) - { - /// Пытаемся найти первую с конца колонку с именем column_name или с именем, начинающимся с column_name и ".". - /// Например "fruits.bananas" - /// одинаковыми считаются имена, если они совпадают целиком или name_without_dot совпадает с частью имени до точки - NamesAndTypesList::reverse_iterator reverse_insert_it = std::find_if(columns.rbegin(), columns.rend(), - std::bind(namesEqual, std::cref(after_column), std::placeholders::_1)); - - if (reverse_insert_it == columns.rend()) - throw Exception("Wrong column name. Cannot find column " + column_name + " to insert after", - DB::ErrorCodes::ILLEGAL_COLUMN); - else - { - /// base возвращает итератор, уже смещенный на один элемент вправо - insert_it = reverse_insert_it.base(); - } - } - - columns.insert(insert_it, NameAndTypePair(column_name, data_type)); - - /// Медленно, так как каждый раз копируется список - columns = *DataTypeNested::expandNestedColumns(columns); - } - else if (type == DROP) - { - bool is_first = true; - NamesAndTypesList::iterator column_it; - do - { - column_it = std::find_if(columns.begin(), columns.end(), - std::bind(namesEqual, std::cref(column_name), std::placeholders::_1)); - - if (column_it == columns.end()) - { - if (is_first) - throw Exception("Wrong column name. Cannot find column " + column_name + " to drop", - DB::ErrorCodes::ILLEGAL_COLUMN); - } - else - columns.erase(column_it); - is_first = false; - } - while (column_it != columns.end()); - } - else if (type == MODIFY) - { - NamesAndTypesList::iterator column_it = std::find_if(columns.begin(), columns.end(), - std::bind(namesEqual, std::cref(column_name), std::placeholders::_1) ); - if (column_it == columns.end()) - throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.", - DB::ErrorCodes::ILLEGAL_COLUMN); - column_it->type = data_type; - } - else - throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR); - } + void apply(NamesAndTypesList & columns, + NamesAndTypesList & materialized_columns, + NamesAndTypesList & alias_columns, + ColumnDefaults & column_defaults) const; }; class AlterCommands : public std::vector { public: - void apply(NamesAndTypesList & columns) const - { - NamesAndTypesList new_columns = columns; - for (const AlterCommand & command : *this) - command.apply(new_columns); - columns = new_columns; - } + void apply(NamesAndTypesList & columns, + NamesAndTypesList & materialized_columns, + NamesAndTypesList & alias_columns, + ColumnDefaults & column_defaults) const; }; } diff --git a/dbms/include/DB/Storages/ColumnDefault.h b/dbms/include/DB/Storages/ColumnDefault.h index c6dd3f986c7..68b667fea26 100644 --- a/dbms/include/DB/Storages/ColumnDefault.h +++ b/dbms/include/DB/Storages/ColumnDefault.h @@ -26,7 +26,7 @@ namespace std namespace DB { - inline ColumnDefaultType columnDefaultTypeFromString(const String& str) + inline ColumnDefaultType columnDefaultTypeFromString(const String & str) { static const std::unordered_map map{ { "DEFAULT", ColumnDefaultType::Default }, diff --git a/dbms/include/DB/Storages/ITableDeclaration.h b/dbms/include/DB/Storages/ITableDeclaration.h index 5379e67ccb9..3af824b1ed3 100644 --- a/dbms/include/DB/Storages/ITableDeclaration.h +++ b/dbms/include/DB/Storages/ITableDeclaration.h @@ -24,7 +24,7 @@ public: /** Получить список имён и типов столбцов таблицы, только невиртуальные. */ NamesAndTypesList getColumnsList() const; - const NamesAndTypesList & getColumnsListAsterisk() const { return getColumnsListImpl(); } + const NamesAndTypesList & getColumnsListNonMaterialized() const { return getColumnsListImpl(); } /** Получить список имён столбцов таблицы, только невиртуальные. */ diff --git a/dbms/include/DB/Storages/StorageChunkRef.h b/dbms/include/DB/Storages/StorageChunkRef.h index 1d8f745cefa..d255d50b67a 100644 --- a/dbms/include/DB/Storages/StorageChunkRef.h +++ b/dbms/include/DB/Storages/StorageChunkRef.h @@ -17,7 +17,7 @@ public: std::string getName() const { return "ChunkRef"; } std::string getTableName() const { return name; } - const NamesAndTypesList & getColumnsListImpl() const override { return getSource().getColumnsListAsterisk(); } + const NamesAndTypesList & getColumnsListImpl() const override { return getSource().getColumnsListNonMaterialized(); } /// В таблице, на которую мы ссылаемся, могут быть виртуальные столбцы. NameAndTypePair getColumn(const String &column_name) const { return getSource().getColumn(column_name); }; bool hasColumn(const String &column_name) const { return getSource().hasColumn(column_name); }; diff --git a/dbms/include/DB/Storages/StorageMergeTree.h b/dbms/include/DB/Storages/StorageMergeTree.h index 7c4c7c4554b..17dfc316eea 100644 --- a/dbms/include/DB/Storages/StorageMergeTree.h +++ b/dbms/include/DB/Storages/StorageMergeTree.h @@ -54,7 +54,7 @@ public: bool supportsFinal() const { return data.supportsFinal(); } bool supportsPrewhere() const { return data.supportsPrewhere(); } - const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListAsterisk(); } + const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListNonMaterialized(); } NameAndTypePair getColumn(const String & column_name) const { diff --git a/dbms/include/DB/Storages/StorageReplicatedMergeTree.h b/dbms/include/DB/Storages/StorageReplicatedMergeTree.h index 33a670b35b0..a8e2856ae69 100644 --- a/dbms/include/DB/Storages/StorageReplicatedMergeTree.h +++ b/dbms/include/DB/Storages/StorageReplicatedMergeTree.h @@ -53,7 +53,7 @@ public: bool supportsFinal() const override { return data.supportsFinal(); } bool supportsPrewhere() const override { return data.supportsPrewhere(); } - const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListAsterisk(); } + const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListNonMaterialized(); } NameAndTypePair getColumn(const String & column_name) const { diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index cc0a0457b35..fdb22ac0c81 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -44,10 +44,11 @@ void Block::addDefaults(NamesAndTypesListPtr required_columns, const auto it = column_defaults.find(column.name); + /// expressions must be cloned to prevent modification by ExpressionAnalyzer if (it == column_defaults.end()) insertDefault(column.name, column.type); else - default_expr_list->children.emplace_back(it->second.expression); + default_expr_list->children.emplace_back(it->second.expression->clone()); } /// nothing to evaluate diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 59890e8d469..5ebe338d5f8 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -35,7 +35,6 @@ void InterpreterAlterQuery::execute() const String & table_name = alter.table; String database_name = alter.database.empty() ? context.getCurrentDatabase() : alter.database; StoragePtr table = context.getTable(database_name, table_name); - validateColumnChanges(alter.parameters, table, context); AlterCommands alter_commands; PartitionCommands partition_commands; @@ -67,11 +66,19 @@ void InterpreterAlterQuery::parseAlter( command.type = AlterCommand::ADD; const auto & ast_col_decl = typeid_cast(*params.col_decl); - StringRange type_range = ast_col_decl.type->range; - String type_string = String(type_range.first, type_range.second - type_range.first); command.column_name = ast_col_decl.name; - command.data_type = data_type_factory.get(type_string); + if (ast_col_decl.type) + { + StringRange type_range = ast_col_decl.type->range; + String type_string(type_range.first, type_range.second - type_range.first); + command.data_type = data_type_factory.get(type_string); + } + if (ast_col_decl.default_expression) + { + command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier); + command.default_expression = setAlias(ast_col_decl.default_expression, ast_col_decl.name); + } if (params.column) command.after_column = typeid_cast(*params.column).name; @@ -92,11 +99,20 @@ void InterpreterAlterQuery::parseAlter( command.type = AlterCommand::MODIFY; const auto & ast_col_decl = typeid_cast(*params.col_decl); - StringRange type_range = ast_col_decl.type->range; - String type_string = String(type_range.first, type_range.second - type_range.first); command.column_name = ast_col_decl.name; - command.data_type = data_type_factory.get(type_string); + if (ast_col_decl.type) + { + StringRange type_range = ast_col_decl.type->range; + String type_string(type_range.first, type_range.second - type_range.first); + command.data_type = data_type_factory.get(type_string); + } + + if (ast_col_decl.default_expression) + { + command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier); + command.default_expression = setAlias(ast_col_decl.default_expression, ast_col_decl.name); + } out_alter_commands.push_back(command); } @@ -115,143 +131,14 @@ void InterpreterAlterQuery::parseAlter( } } -void InterpreterAlterQuery::validateColumnChanges(ASTAlterQuery::ParameterContainer & params_container, const StoragePtr & table, const Context & context) -{ - auto columns = table->getColumnsList(); - columns.insert(std::end(columns), std::begin(table->materialized_columns), std::end(table->materialized_columns)); - columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns)); - auto defaults = table->column_defaults; - - std::vector> defaulted_columns{}; - - ASTPtr default_expr_list{new ASTExpressionList}; - default_expr_list->children.reserve(table->column_defaults.size()); - - for (auto & params : params_container) - { - if (params.type == ASTAlterQuery::ADD_COLUMN || params.type == ASTAlterQuery::MODIFY_COLUMN) - { - auto & col_decl = typeid_cast(*params.col_decl); - const auto & column_name = col_decl.name; - - if (params.type == ASTAlterQuery::MODIFY_COLUMN) - { - const auto it = std::find_if(std::begin(columns), std::end(columns), - std::bind(AlterCommand::namesEqual, std::cref(column_name), std::placeholders::_1)); - - if (it == std::end(columns)) - throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.", - DB::ErrorCodes::ILLEGAL_COLUMN); - - columns.erase(it); - defaults.erase(column_name); - } - - if (col_decl.type) - { - const StringRange & type_range = col_decl.type->range; - columns.emplace_back(col_decl.name, - context.getDataTypeFactory().get({type_range.first, type_range.second})); - } - - if (col_decl.default_expression) - { - - if (col_decl.type) - { - const auto tmp_column_name = col_decl.name + "_tmp"; - const auto & final_column_name = col_decl.name; - const auto conversion_function_name = "to" + columns.back().type->getName(); - - default_expr_list->children.emplace_back(setAlias( - makeASTFunction(conversion_function_name, ASTPtr{new ASTIdentifier{{}, tmp_column_name}}), - final_column_name)); - - default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), tmp_column_name)); - - defaulted_columns.emplace_back(columns.back().type.get(), &col_decl); - } - else - { - default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), col_decl.name)); - - defaulted_columns.emplace_back(nullptr, &col_decl); - } - } - } - else if (params.type == ASTAlterQuery::DROP_COLUMN) - { - const auto & column_name = typeid_cast(*(params.column)).name; - - auto found = false; - for (auto it = std::begin(columns); it != std::end(columns);) - if (AlterCommand::namesEqual(column_name, *it)) - { - found = true; - it = columns.erase(it); - } - else - ++it; - - for (auto it = std::begin(defaults); it != std::end(defaults);) - if (AlterCommand::namesEqual(column_name, { it->first, nullptr })) - it = defaults.erase(it); - else - ++it; - - if (!found) - throw Exception("Wrong column name. Cannot find column " + column_name + " to drop.", - DB::ErrorCodes::ILLEGAL_COLUMN); - } - } - - for (const auto & col_def : defaults) - default_expr_list->children.emplace_back(setAlias(col_def.second.expression->clone(), col_def.first)); - - const auto actions = ExpressionAnalyzer{default_expr_list, context, columns}.getActions(true); - const auto block = actions->getSampleBlock(); - - for (auto & column : defaulted_columns) - { - const auto type_ptr = column.first; - const auto col_decl_ptr = column.second; - - if (type_ptr) - { - const auto & tmp_column = block.getByName(col_decl_ptr->name + "_tmp"); - - /// type mismatch between explicitly specified and deduced type, add conversion - if (typeid(*type_ptr) != typeid(*tmp_column.type)) - { - col_decl_ptr->default_expression = makeASTFunction( - "to" + type_ptr->getName(), - col_decl_ptr->default_expression); - - col_decl_ptr->children.clear(); - col_decl_ptr->children.push_back(col_decl_ptr->type); - col_decl_ptr->children.push_back(col_decl_ptr->default_expression); - } - } - else - { - col_decl_ptr->type = new ASTIdentifier{}; - col_decl_ptr->query_string = new String{block.getByName(col_decl_ptr->name).type->getName()}; - col_decl_ptr->range = { - col_decl_ptr->query_string->data(), - col_decl_ptr->query_string->data() + col_decl_ptr->query_string->size() - }; - static_cast(*col_decl_ptr->type).name = *col_decl_ptr->query_string; - } - - defaults.emplace(col_decl_ptr->name, ColumnDefault{ - columnDefaultTypeFromString(col_decl_ptr->default_specifier), - setAlias(col_decl_ptr->default_expression, col_decl_ptr->name) - }); - } -} - void InterpreterAlterQuery::updateMetadata( - const String & database_name, const String & table_name, const NamesAndTypesList & columns, Context & context) + const String & database_name, + const String & table_name, + const NamesAndTypesList & columns, + const NamesAndTypesList & materialized_columns, + const NamesAndTypesList & alias_columns, + const ColumnDefaults & column_defaults, + Context & context) { String path = context.getPath(); @@ -286,7 +173,7 @@ void InterpreterAlterQuery::updateMetadata( ASTCreateQuery & attach = typeid_cast(*ast); - ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns); + ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns, materialized_columns, alias_columns, column_defaults); *std::find(attach.children.begin(), attach.children.end(), attach.columns) = new_columns; attach.columns = new_columns; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 1192c4908f9..5564de6774d 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -128,7 +128,7 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists) } else if (!create.as_table.empty()) { - columns = new NamesAndTypesList(as_storage->getColumnsListAsterisk()); + columns = new NamesAndTypesList(as_storage->getColumnsListNonMaterialized()); materialized_columns = as_storage->materialized_columns; alias_columns = as_storage->alias_columns; column_defaults = as_storage->column_defaults; diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 0e24558adb4..161e476492b 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -67,7 +67,7 @@ void InterpreterSelectQuery::init(BlockInputStreamPtr input_, const NamesAndType table_lock = storage->lockStructure(false); if (table_column_names.empty()) - context.setColumns(storage->getColumnsListAsterisk()); + context.setColumns(storage->getColumnsListNonMaterialized()); } if (!table_column_names.empty()) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp new file mode 100644 index 00000000000..1bae4079ac1 --- /dev/null +++ b/dbms/src/Storages/AlterCommands.cpp @@ -0,0 +1,151 @@ +#include +#include +#include + +namespace DB +{ + void AlterCommand::apply(NamesAndTypesList & columns, + NamesAndTypesList & materialized_columns, + NamesAndTypesList & alias_columns, + ColumnDefaults & column_defaults) const + { + if (type == ADD) + { + const auto throw_if_exists = [this] (const NamesAndTypesList & columns) { + if (std::count_if(columns.begin(), columns.end(), + std::bind(namesEqual, std::cref(column_name), std::placeholders::_1))) + { + throw Exception("Cannot add column " + column_name + ": column with this name already exisits.", + DB::ErrorCodes::ILLEGAL_COLUMN); + } + }; + + throw_if_exists(columns); + throw_if_exists(materialized_columns); + throw_if_exists(alias_columns); + + const auto add_column = [this] (NamesAndTypesList & columns) { + auto insert_it = columns.end(); + + if (!after_column.empty()) + { + /// Пытаемся найти первую с конца колонку с именем column_name или с именем, начинающимся с column_name и ".". + /// Например "fruits.bananas" + /// одинаковыми считаются имена, если они совпадают целиком или name_without_dot совпадает с частью имени до точки + const auto reverse_insert_it = std::find_if(columns.rbegin(), columns.rend(), + std::bind(namesEqual, std::cref(after_column), std::placeholders::_1)); + + if (reverse_insert_it == columns.rend()) + throw Exception("Wrong column name. Cannot find column " + column_name + " to insert after", + DB::ErrorCodes::ILLEGAL_COLUMN); + else + { + /// base возвращает итератор, уже смещенный на один элемент вправо + insert_it = reverse_insert_it.base(); + } + } + + columns.emplace(insert_it, column_name, data_type); + }; + + if (default_type == ColumnDefaultType::Default) + add_column(columns); + else if (default_type == ColumnDefaultType::Materialized) + add_column(materialized_columns); + else if (default_type == ColumnDefaultType::Alias) + add_column(alias_columns); + else + throw Exception{"Unknown ColumnDefaultType value", ErrorCodes::LOGICAL_ERROR}; + + if (default_expression) + column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); + + /// Медленно, так как каждый раз копируется список + columns = *DataTypeNested::expandNestedColumns(columns); + } + else if (type == DROP) + { + /// look for a column in list and remove it if present, also removing corresponding entry from column_defaults + const auto remove_column = [&column_defaults, this] (NamesAndTypesList & columns) { + auto removed = false; + NamesAndTypesList::iterator column_it; + + while (columns.end() != (column_it = std::find_if(columns.begin(), columns.end(), + std::bind(namesEqual, std::cref(column_name), std::placeholders::_1)))) + { + removed = true; + column_it = columns.erase(column_it); + column_defaults.erase(column_name); + } + + return removed; + }; + + if (!remove_column(columns) && + !remove_column(materialized_columns) && + !remove_column(alias_columns)) + { + throw Exception("Wrong column name. Cannot find column " + column_name + " to drop", + DB::ErrorCodes::ILLEGAL_COLUMN); + } + } + else if (type == MODIFY) + { + const auto it = column_defaults.find(column_name); + const auto had_default_expr = it != column_defaults.end(); + const auto old_default_type = had_default_expr ? it->second.type : ColumnDefaultType{}; + + if (old_default_type != default_type) + throw Exception{"Cannot change column default specifier from " + toString(old_default_type) + + " to " + toString(default_type), 0 }; + + /// find column or throw exception + const auto find_column = [this] (NamesAndTypesList & columns) { + const auto it = std::find_if(columns.begin(), columns.end(), + std::bind(namesEqual, std::cref(column_name), std::placeholders::_1) ); + if (it == columns.end()) + throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.", + DB::ErrorCodes::ILLEGAL_COLUMN); + + return it; + }; + + /// find column in one of three column lists + const auto column_it = find_column( + default_type == ColumnDefaultType::Default ? columns : + default_type == ColumnDefaultType::Materialized ? materialized_columns : + alias_columns); + + column_it->type = data_type; + + /// remove, add or update default_expression + if (!default_expression && had_default_expr) + column_defaults.erase(column_name); + else if (default_expression && !had_default_expr) + column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); + else + column_defaults[column_name].expression = default_expression; + } + else + throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR); + } + + void AlterCommands::apply(NamesAndTypesList & columns, + NamesAndTypesList & materialized_columns, + NamesAndTypesList & alias_columns, + ColumnDefaults & column_defaults) const + { + auto new_columns = columns; + auto new_materialized_columns = materialized_columns; + auto new_alias_columns = alias_columns; + auto new_column_defaults = column_defaults; + + for (const AlterCommand & command : *this) + command.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); + + columns = std::move(new_columns); + materialized_columns = std::move(new_materialized_columns); + alias_columns = std::move(new_alias_columns); + column_defaults = std::move(new_column_defaults); + } +} diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index 0a6144b20b8..2621180b24c 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -103,7 +103,7 @@ Block ITableDeclaration::getSampleBlockNonMaterialized() const { Block res; - for (const auto & col : getColumnsListAsterisk()) + for (const auto & col : getColumnsListNonMaterialized()) res.insert({ col.type->createColumn(), col.type, col.name }); return res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index ba3eb9e25ee..ff1113afd98 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -360,8 +360,11 @@ void MergeTreeData::dropAllData() void MergeTreeData::checkAlter(const AlterCommands & params) { /// Проверим, что указанные преобразования можно совершить над списком столбцов без учета типов. - NamesAndTypesList new_columns = *columns; - params.apply(new_columns); + auto new_columns = *columns; + auto new_materialized_columns = materialized_columns; + auto new_alias_columns = alias_columns; + auto new_column_defaults = column_defaults; + params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); /// Список столбцов, которые нельзя трогать. /// sampling_expression можно не учитывать, потому что он обязан содержаться в первичном ключе. diff --git a/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp b/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp index ef1b36776fd..69c124e3675 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 98d0785907e..f1701bdefb4 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -203,8 +203,9 @@ BlockOutputStreamPtr StorageDistributed::write(ASTPtr query) void StorageDistributed::alter(const AlterCommands & params, const String & database_name, const String & table_name, Context & context) { auto lock = lockStructureForAlter(); - params.apply(*columns); - InterpreterAlterQuery::updateMetadata(database_name, table_name, *columns, context); + params.apply(*columns, materialized_columns, alias_columns, column_defaults); + InterpreterAlterQuery::updateMetadata(database_name, table_name, + *columns, materialized_columns, alias_columns, column_defaults, context); } void StorageDistributed::shutdown() diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 13409135104..a3d03eb6f14 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -203,8 +203,9 @@ void StorageMerge::getSelectedTables(StorageVector & selected_tables) const void StorageMerge::alter(const AlterCommands & params, const String & database_name, const String & table_name, Context & context) { auto lock = lockStructureForAlter(); - params.apply(*columns); - InterpreterAlterQuery::updateMetadata(database_name, table_name, *columns, context); + params.apply(*columns, materialized_columns, alias_columns, column_defaults); + InterpreterAlterQuery::updateMetadata(database_name, table_name, *columns, + materialized_columns, alias_columns, column_defaults, context); } } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 9807e1738f8..ec05541a8f2 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -129,8 +129,12 @@ void StorageMergeTree::alter(const AlterCommands & params, const String & databa data.checkAlter(params); - NamesAndTypesList new_columns = data.getColumnsList(); - params.apply(new_columns); + auto new_columns = data.getColumnsListNonMaterialized(); + auto new_materialized_columns = data.materialized_columns; + auto new_alias_columns = data.alias_columns; + auto new_column_defaults = data.column_defaults; + + params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); MergeTreeData::DataParts parts = data.getDataParts(); std::vector transactions; @@ -143,8 +147,17 @@ void StorageMergeTree::alter(const AlterCommands & params, const String & databa auto table_hard_lock = lockStructureForAlter(); - InterpreterAlterQuery::updateMetadata(database_name, table_name, new_columns, context); + InterpreterAlterQuery::updateMetadata(database_name, table_name, new_columns, + new_materialized_columns, new_alias_columns, new_column_defaults, context); + + materialized_columns = new_materialized_columns; + alias_columns = new_alias_columns; + column_defaults = new_column_defaults; + data.setColumnsList(new_columns); + data.materialized_columns = std::move(new_materialized_columns); + data.alias_columns = std::move(new_alias_columns); + data.column_defaults = std::move(new_column_defaults); for (auto & transaction : transactions) { diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 088f302b57d..e5f21ee1b86 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -207,7 +207,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists() zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); ops.push_back(new zkutil::Op::Create(zookeeper_path + "/metadata", metadata.str(), zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); - ops.push_back(new zkutil::Op::Create(zookeeper_path + "/columns", data.getColumnsList().toString(), + ops.push_back(new zkutil::Op::Create(zookeeper_path + "/columns", data.getColumnsListNonMaterialized().toString(), zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); ops.push_back(new zkutil::Op::Create(zookeeper_path + "/log", "", zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); @@ -256,16 +256,23 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo zkutil::Stat stat; auto columns = NamesAndTypesList::parse(zookeeper->get(zookeeper_path + "/columns", &stat), context.getDataTypeFactory()); + NamesAndTypesList materialized_columns; + NamesAndTypesList alias_columns; + ColumnDefaults column_defaults; columns_version = stat.version; - if (columns != data.getColumnsList()) + if (columns != data.getColumnsListNonMaterialized()) { - if (allow_alter && (data.getColumnsList().sizeOfDifference(columns) <= 2 || skip_sanity_checks)) + if (allow_alter && (data.getColumnsListNonMaterialized().sizeOfDifference(columns) <= 2 || skip_sanity_checks)) { LOG_WARNING(log, "Table structure in ZooKeeper is a little different from local table structure. Assuming ALTER."); /// Без всяких блокировок, потому что таблица еще не создана. - InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, context); + InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, + materialized_columns, alias_columns, column_defaults, context); data.setColumnsList(columns); + data.materialized_columns = std::move(materialized_columns); + data.alias_columns = std::move(alias_columns); + data.column_defaults = std::move(column_defaults); } else { @@ -403,7 +410,7 @@ void StorageReplicatedMergeTree::createReplica() LOG_DEBUG(log, "Copied " << source_queue.size() << " queue entries"); } - zookeeper->create(replica_path + "/columns", data.getColumnsList().toString(), zkutil::CreateMode::Persistent); + zookeeper->create(replica_path + "/columns", data.getColumnsListNonMaterialized().toString(), zkutil::CreateMode::Persistent); } void StorageReplicatedMergeTree::activateReplica() @@ -1583,6 +1590,9 @@ void StorageReplicatedMergeTree::alterThread() zkutil::Stat stat; String columns_str = zookeeper->get(zookeeper_path + "/columns", &stat, alter_thread_event); NamesAndTypesList columns = NamesAndTypesList::parse(columns_str, context.getDataTypeFactory()); + NamesAndTypesList materialized_columns; + NamesAndTypesList alias_columns; + ColumnDefaults column_defaults; bool changed_version = (stat.version != columns_version); @@ -1593,12 +1603,22 @@ void StorageReplicatedMergeTree::alterThread() { auto table_lock = lockStructureForAlter(); - if (columns != data.getColumnsList()) + if (columns != data.getColumnsListNonMaterialized()) { LOG_INFO(log, "Columns list changed in ZooKeeper. Applying changes locally."); - InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, context); + InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, + materialized_columns, alias_columns, column_defaults, context); + + this->materialized_columns = materialized_columns; + this->alias_columns = alias_columns; + this->column_defaults = column_defaults; + data.setColumnsList(columns); + data.materialized_columns = std::move(materialized_columns); + data.alias_columns = std::move(alias_columns); + data.column_defaults = std::move(column_defaults); + if (unreplicated_data) unreplicated_data->setColumnsList(columns); LOG_INFO(log, "Applied changes to table."); @@ -2306,6 +2326,9 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, LOG_DEBUG(log, "Doing ALTER"); NamesAndTypesList new_columns; + NamesAndTypesList new_materialized_columns; + NamesAndTypesList new_alias_columns; + ColumnDefaults new_column_defaults; String new_columns_str; int new_columns_version; zkutil::Stat stat; @@ -2318,8 +2341,11 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, data.checkAlter(params); - new_columns = data.getColumnsList(); - params.apply(new_columns); + new_columns = data.getColumnsListNonMaterialized(); + new_materialized_columns = data.materialized_columns; + new_alias_columns = data.alias_columns; + new_column_defaults = data.column_defaults; + params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); new_columns_str = new_columns.toString(); diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 156385a61c1..bb059a299da 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -36,7 +36,7 @@ StorageView::StorageView( const NamesAndTypesList & materialized_columns_, const NamesAndTypesList & alias_columns_, const ColumnDefaults & column_defaults_) - : IStorage{materialized_columns_, alias_columns_, column_defaults}, table_name(table_name_), + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, table_name(table_name_), database_name(database_name_), context(context_), columns(columns_) { ASTCreateQuery & create = typeid_cast(*query_);