thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

dbms: defaulted columns support for ALTER, currently without type deduction and checking.

Browse Source
This commit is contained in:
Andrey Mironov 2014-10-16 17:37:01 +04:00
parent 43deee45c9
commit 0bfd35e5c7
20 changed files with 278 additions and 254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dbms/include/DB/Interpreters/InterpreterAlterQuery.h
View File

@ -21,7 +21,13 @@ public:
/** Изменяет список столбцов в метаданных таблицы на диске. Нужно вызывать под TableStructureLock соответствующей таблицы.
*/
static void updateMetadata(const String & database, const String & table, const NamesAndTypesList & columns, Context & context);
static void updateMetadata(const String & database,
const String & table,
const NamesAndTypesList & columns,
const NamesAndTypesList & materialized_columns,
const NamesAndTypesList & alias_columns,
const ColumnDefaults & column_defaults,
Context & context);
private:
struct PartitionCommand
{
@ -58,7 +64,6 @@ private:
static void parseAlter(const ASTAlterQuery::ParameterContainer & params, const DataTypeFactory & data_type_factory,
AlterCommands & out_alter_commands, PartitionCommands & out_partition_commands);
static void validateColumnChanges(ASTAlterQuery::ParameterContainer & params, const StoragePtr & table, const Context & context);
};
}

92
dbms/include/DB/Storages/AlterCommands.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <DB/Core/NamesAndTypes.h>
#include <DB/DataTypes/DataTypeNested.h>
#include <DB/DataTypes/DataTypeArray.h>
#include <DB/Storages/ColumnDefault.h>
namespace DB
{
@ -23,6 +23,9 @@ struct AlterCommand
/// Для ADD и MODIFY - новый тип столбца.
DataTypePtr data_type;
ColumnDefaultType default_type{};
ASTPtr default_expression{};
/// Для ADD - после какого столбца добавить новый. Если пустая строка, добавить в конец. Добавить в начало сейчас нельзя.
String after_column;
@ -34,88 +37,19 @@ struct AlterCommand
return (name_with_dot == name_type.name.substr(0, name_without_dot.length() + 1) || name_without_dot == name_type.name);
}
void apply(NamesAndTypesList & columns) const
{
if (type == ADD)
{
if (std::count_if(columns.begin(), columns.end(), std::bind(namesEqual, std::cref(column_name), std::placeholders::_1)))
throw Exception("Cannot add column " + column_name + ": column with this name already exisits.",
DB::ErrorCodes::ILLEGAL_COLUMN);
if (DataTypeNested::extractNestedTableName(column_name) != column_name &&
!typeid_cast<const DataTypeArray *>(&*data_type))
throw Exception("Can't add nested column " + column_name + " of non-array type " + data_type->getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
NamesAndTypesList::iterator insert_it = columns.end();
if (!after_column.empty())
{
/// Пытаемся найти первую с конца колонку с именем column_name или с именем, начинающимся с column_name и ".".
/// Например "fruits.bananas"
/// одинаковыми считаются имена, если они совпадают целиком или name_without_dot совпадает с частью имени до точки
NamesAndTypesList::reverse_iterator reverse_insert_it = std::find_if(columns.rbegin(), columns.rend(),
std::bind(namesEqual, std::cref(after_column), std::placeholders::_1));
if (reverse_insert_it == columns.rend())
throw Exception("Wrong column name. Cannot find column " + column_name + " to insert after",
DB::ErrorCodes::ILLEGAL_COLUMN);
else
{
/// base возвращает итератор, уже смещенный на один элемент вправо
insert_it = reverse_insert_it.base();
}
}
columns.insert(insert_it, NameAndTypePair(column_name, data_type));
/// Медленно, так как каждый раз копируется список
columns = *DataTypeNested::expandNestedColumns(columns);
}
else if (type == DROP)
{
bool is_first = true;
NamesAndTypesList::iterator column_it;
do
{
column_it = std::find_if(columns.begin(), columns.end(),
std::bind(namesEqual, std::cref(column_name), std::placeholders::_1));
if (column_it == columns.end())
{
if (is_first)
throw Exception("Wrong column name. Cannot find column " + column_name + " to drop",
DB::ErrorCodes::ILLEGAL_COLUMN);
}
else
columns.erase(column_it);
is_first = false;
}
while (column_it != columns.end());
}
else if (type == MODIFY)
{
NamesAndTypesList::iterator column_it = std::find_if(columns.begin(), columns.end(),
std::bind(namesEqual, std::cref(column_name), std::placeholders::_1) );
if (column_it == columns.end())
throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.",
DB::ErrorCodes::ILLEGAL_COLUMN);
column_it->type = data_type;
}
else
throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR);
}
void apply(NamesAndTypesList & columns,
NamesAndTypesList & materialized_columns,
NamesAndTypesList & alias_columns,
ColumnDefaults & column_defaults) const;
};
class AlterCommands : public std::vector<AlterCommand>
{
public:
void apply(NamesAndTypesList & columns) const
{
NamesAndTypesList new_columns = columns;
for (const AlterCommand & command : *this)
command.apply(new_columns);
columns = new_columns;
}
void apply(NamesAndTypesList & columns,
NamesAndTypesList & materialized_columns,
NamesAndTypesList & alias_columns,
ColumnDefaults & column_defaults) const;
};
}

2
dbms/include/DB/Storages/ColumnDefault.h
View File

@ -26,7 +26,7 @@ namespace std
namespace DB
{
inline ColumnDefaultType columnDefaultTypeFromString(const String& str)
inline ColumnDefaultType columnDefaultTypeFromString(const String & str)
{
static const std::unordered_map<String, ColumnDefaultType> map{
{ "DEFAULT", ColumnDefaultType::Default },

2
dbms/include/DB/Storages/ITableDeclaration.h
View File

@ -24,7 +24,7 @@ public:
/** Получить список имён и типов столбцов таблицы, только невиртуальные.
*/
NamesAndTypesList getColumnsList() const;
const NamesAndTypesList & getColumnsListAsterisk() const { return getColumnsListImpl(); }
const NamesAndTypesList & getColumnsListNonMaterialized() const { return getColumnsListImpl(); }
/** Получить список имён столбцов таблицы, только невиртуальные.
*/

2
dbms/include/DB/Storages/StorageChunkRef.h
View File

@ -17,7 +17,7 @@ public:
std::string getName() const { return "ChunkRef"; }
std::string getTableName() const { return name; }
const NamesAndTypesList & getColumnsListImpl() const override { return getSource().getColumnsListAsterisk(); }
const NamesAndTypesList & getColumnsListImpl() const override { return getSource().getColumnsListNonMaterialized(); }
/// В таблице, на которую мы ссылаемся, могут быть виртуальные столбцы.
NameAndTypePair getColumn(const String &column_name) const { return getSource().getColumn(column_name); };
bool hasColumn(const String &column_name) const { return getSource().hasColumn(column_name); };

2
dbms/include/DB/Storages/StorageMergeTree.h
View File

@ -54,7 +54,7 @@ public:
bool supportsFinal() const { return data.supportsFinal(); }
bool supportsPrewhere() const { return data.supportsPrewhere(); }
const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListAsterisk(); }
const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListNonMaterialized(); }
NameAndTypePair getColumn(const String & column_name) const
{

2
dbms/include/DB/Storages/StorageReplicatedMergeTree.h
View File

@ -53,7 +53,7 @@ public:
bool supportsFinal() const override { return data.supportsFinal(); }
bool supportsPrewhere() const override { return data.supportsPrewhere(); }
const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListAsterisk(); }
const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListNonMaterialized(); }
NameAndTypePair getColumn(const String & column_name) const
{

3
dbms/src/Core/Block.cpp
View File

@ -44,10 +44,11 @@ void Block::addDefaults(NamesAndTypesListPtr required_columns,
const auto it = column_defaults.find(column.name);
/// expressions must be cloned to prevent modification by ExpressionAnalyzer
if (it == column_defaults.end())
insertDefault(column.name, column.type);
else
default_expr_list->children.emplace_back(it->second.expression);
default_expr_list->children.emplace_back(it->second.expression->clone());
}
/// nothing to evaluate

175
dbms/src/Interpreters/InterpreterAlterQuery.cpp
View File

@ -35,7 +35,6 @@ void InterpreterAlterQuery::execute()
const String & table_name = alter.table;
String database_name = alter.database.empty() ? context.getCurrentDatabase() : alter.database;
StoragePtr table = context.getTable(database_name, table_name);
validateColumnChanges(alter.parameters, table, context);
AlterCommands alter_commands;
PartitionCommands partition_commands;
@ -67,11 +66,19 @@ void InterpreterAlterQuery::parseAlter(
command.type = AlterCommand::ADD;
const auto & ast_col_decl = typeid_cast<const ASTColumnDeclaration &>(*params.col_decl);
StringRange type_range = ast_col_decl.type->range;
String type_string = String(type_range.first, type_range.second - type_range.first);
command.column_name = ast_col_decl.name;
command.data_type = data_type_factory.get(type_string);
if (ast_col_decl.type)
{
StringRange type_range = ast_col_decl.type->range;
String type_string(type_range.first, type_range.second - type_range.first);
command.data_type = data_type_factory.get(type_string);
}
if (ast_col_decl.default_expression)
{
command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier);
command.default_expression = setAlias(ast_col_decl.default_expression, ast_col_decl.name);
}
if (params.column)
command.after_column = typeid_cast<const ASTIdentifier &>(*params.column).name;
@ -92,11 +99,20 @@ void InterpreterAlterQuery::parseAlter(
command.type = AlterCommand::MODIFY;
const auto & ast_col_decl = typeid_cast<const ASTColumnDeclaration &>(*params.col_decl);
StringRange type_range = ast_col_decl.type->range;
String type_string = String(type_range.first, type_range.second - type_range.first);
command.column_name = ast_col_decl.name;
command.data_type = data_type_factory.get(type_string);
if (ast_col_decl.type)
{
StringRange type_range = ast_col_decl.type->range;
String type_string(type_range.first, type_range.second - type_range.first);
command.data_type = data_type_factory.get(type_string);
}
if (ast_col_decl.default_expression)
{
command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier);
command.default_expression = setAlias(ast_col_decl.default_expression, ast_col_decl.name);
}
out_alter_commands.push_back(command);
}
@ -115,143 +131,14 @@ void InterpreterAlterQuery::parseAlter(
}
}
void InterpreterAlterQuery::validateColumnChanges(ASTAlterQuery::ParameterContainer & params_container, const StoragePtr & table, const Context & context)
{
auto columns = table->getColumnsList();
columns.insert(std::end(columns), std::begin(table->materialized_columns), std::end(table->materialized_columns));
columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns));
auto defaults = table->column_defaults;
std::vector<std::pair<IDataType *, ASTColumnDeclaration *>> defaulted_columns{};
ASTPtr default_expr_list{new ASTExpressionList};
default_expr_list->children.reserve(table->column_defaults.size());
for (auto & params : params_container)
{
if (params.type == ASTAlterQuery::ADD_COLUMN || params.type == ASTAlterQuery::MODIFY_COLUMN)
{
auto & col_decl = typeid_cast<ASTColumnDeclaration &>(*params.col_decl);
const auto & column_name = col_decl.name;
if (params.type == ASTAlterQuery::MODIFY_COLUMN)
{
const auto it = std::find_if(std::begin(columns), std::end(columns),
std::bind(AlterCommand::namesEqual, std::cref(column_name), std::placeholders::_1));
if (it == std::end(columns))
throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.",
DB::ErrorCodes::ILLEGAL_COLUMN);
columns.erase(it);
defaults.erase(column_name);
}
if (col_decl.type)
{
const StringRange & type_range = col_decl.type->range;
columns.emplace_back(col_decl.name,
context.getDataTypeFactory().get({type_range.first, type_range.second}));
}
if (col_decl.default_expression)
{
if (col_decl.type)
{
const auto tmp_column_name = col_decl.name + "_tmp";
const auto & final_column_name = col_decl.name;
const auto conversion_function_name = "to" + columns.back().type->getName();
default_expr_list->children.emplace_back(setAlias(
makeASTFunction(conversion_function_name, ASTPtr{new ASTIdentifier{{}, tmp_column_name}}),
final_column_name));
default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), tmp_column_name));
defaulted_columns.emplace_back(columns.back().type.get(), &col_decl);
}
else
{
default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), col_decl.name));
defaulted_columns.emplace_back(nullptr, &col_decl);
}
}
}
else if (params.type == ASTAlterQuery::DROP_COLUMN)
{
const auto & column_name = typeid_cast<const ASTIdentifier &>(*(params.column)).name;
auto found = false;
for (auto it = std::begin(columns); it != std::end(columns);)
if (AlterCommand::namesEqual(column_name, *it))
{
found = true;
it = columns.erase(it);
}
else
++it;
for (auto it = std::begin(defaults); it != std::end(defaults);)
if (AlterCommand::namesEqual(column_name, { it->first, nullptr }))
it = defaults.erase(it);
else
++it;
if (!found)
throw Exception("Wrong column name. Cannot find column " + column_name + " to drop.",
DB::ErrorCodes::ILLEGAL_COLUMN);
}
}
for (const auto & col_def : defaults)
default_expr_list->children.emplace_back(setAlias(col_def.second.expression->clone(), col_def.first));
const auto actions = ExpressionAnalyzer{default_expr_list, context, columns}.getActions(true);
const auto block = actions->getSampleBlock();
for (auto & column : defaulted_columns)
{
const auto type_ptr = column.first;
const auto col_decl_ptr = column.second;
if (type_ptr)
{
const auto & tmp_column = block.getByName(col_decl_ptr->name + "_tmp");
/// type mismatch between explicitly specified and deduced type, add conversion
if (typeid(*type_ptr) != typeid(*tmp_column.type))
{
col_decl_ptr->default_expression = makeASTFunction(
"to" + type_ptr->getName(),
col_decl_ptr->default_expression);
col_decl_ptr->children.clear();
col_decl_ptr->children.push_back(col_decl_ptr->type);
col_decl_ptr->children.push_back(col_decl_ptr->default_expression);
}
}
else
{
col_decl_ptr->type = new ASTIdentifier{};
col_decl_ptr->query_string = new String{block.getByName(col_decl_ptr->name).type->getName()};
col_decl_ptr->range = {
col_decl_ptr->query_string->data(),
col_decl_ptr->query_string->data() + col_decl_ptr->query_string->size()
};
static_cast<ASTIdentifier &>(*col_decl_ptr->type).name = *col_decl_ptr->query_string;
}
defaults.emplace(col_decl_ptr->name, ColumnDefault{
columnDefaultTypeFromString(col_decl_ptr->default_specifier),
setAlias(col_decl_ptr->default_expression, col_decl_ptr->name)
});
}
}
void InterpreterAlterQuery::updateMetadata(
const String & database_name, const String & table_name, const NamesAndTypesList & columns, Context & context)
const String & database_name,
const String & table_name,
const NamesAndTypesList & columns,
const NamesAndTypesList & materialized_columns,
const NamesAndTypesList & alias_columns,
const ColumnDefaults & column_defaults,
Context & context)
{
String path = context.getPath();
@ -286,7 +173,7 @@ void InterpreterAlterQuery::updateMetadata(
ASTCreateQuery & attach = typeid_cast<ASTCreateQuery &>(*ast);
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns);
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns, materialized_columns, alias_columns, column_defaults);
*std::find(attach.children.begin(), attach.children.end(), attach.columns) = new_columns;
attach.columns = new_columns;

2
dbms/src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -128,7 +128,7 @@ StoragePtr InterpreterCreateQuery::execute(bool assume_metadata_exists)
}
else if (!create.as_table.empty())
{
columns = new NamesAndTypesList(as_storage->getColumnsListAsterisk());
columns = new NamesAndTypesList(as_storage->getColumnsListNonMaterialized());
materialized_columns = as_storage->materialized_columns;
alias_columns = as_storage->alias_columns;
column_defaults = as_storage->column_defaults;

2
dbms/src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -67,7 +67,7 @@ void InterpreterSelectQuery::init(BlockInputStreamPtr input_, const NamesAndType
table_lock = storage->lockStructure(false);
if (table_column_names.empty())
context.setColumns(storage->getColumnsListAsterisk());
context.setColumns(storage->getColumnsListNonMaterialized());
}
if (!table_column_names.empty())

151
dbms/src/Storages/AlterCommands.cpp Normal file
View File

@ -0,0 +1,151 @@
#include <DB/Storages/AlterCommands.h>
#include <DB/DataTypes/DataTypeNested.h>
#include <DB/DataTypes/DataTypeArray.h>
namespace DB
{
void AlterCommand::apply(NamesAndTypesList & columns,
NamesAndTypesList & materialized_columns,
NamesAndTypesList & alias_columns,
ColumnDefaults & column_defaults) const
{
if (type == ADD)
{
const auto throw_if_exists = [this] (const NamesAndTypesList & columns) {
if (std::count_if(columns.begin(), columns.end(),
std::bind(namesEqual, std::cref(column_name), std::placeholders::_1)))
{
throw Exception("Cannot add column " + column_name + ": column with this name already exisits.",
DB::ErrorCodes::ILLEGAL_COLUMN);
}
};
throw_if_exists(columns);
throw_if_exists(materialized_columns);
throw_if_exists(alias_columns);
const auto add_column = [this] (NamesAndTypesList & columns) {
auto insert_it = columns.end();
if (!after_column.empty())
{
/// Пытаемся найти первую с конца колонку с именем column_name или с именем, начинающимся с column_name и ".".
/// Например "fruits.bananas"
/// одинаковыми считаются имена, если они совпадают целиком или name_without_dot совпадает с частью имени до точки
const auto reverse_insert_it = std::find_if(columns.rbegin(), columns.rend(),
std::bind(namesEqual, std::cref(after_column), std::placeholders::_1));
if (reverse_insert_it == columns.rend())
throw Exception("Wrong column name. Cannot find column " + column_name + " to insert after",
DB::ErrorCodes::ILLEGAL_COLUMN);
else
{
/// base возвращает итератор, уже смещенный на один элемент вправо
insert_it = reverse_insert_it.base();
}
}
columns.emplace(insert_it, column_name, data_type);
};
if (default_type == ColumnDefaultType::Default)
add_column(columns);
else if (default_type == ColumnDefaultType::Materialized)
add_column(materialized_columns);
else if (default_type == ColumnDefaultType::Alias)
add_column(alias_columns);
else
throw Exception{"Unknown ColumnDefaultType value", ErrorCodes::LOGICAL_ERROR};
if (default_expression)
column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression});
/// Медленно, так как каждый раз копируется список
columns = *DataTypeNested::expandNestedColumns(columns);
}
else if (type == DROP)
{
/// look for a column in list and remove it if present, also removing corresponding entry from column_defaults
const auto remove_column = [&column_defaults, this] (NamesAndTypesList & columns) {
auto removed = false;
NamesAndTypesList::iterator column_it;
while (columns.end() != (column_it = std::find_if(columns.begin(), columns.end(),
std::bind(namesEqual, std::cref(column_name), std::placeholders::_1))))
{
removed = true;
column_it = columns.erase(column_it);
column_defaults.erase(column_name);
}
return removed;
};
if (!remove_column(columns) &&
!remove_column(materialized_columns) &&
!remove_column(alias_columns))
{
throw Exception("Wrong column name. Cannot find column " + column_name + " to drop",
DB::ErrorCodes::ILLEGAL_COLUMN);
}
}
else if (type == MODIFY)
{
const auto it = column_defaults.find(column_name);
const auto had_default_expr = it != column_defaults.end();
const auto old_default_type = had_default_expr ? it->second.type : ColumnDefaultType{};
if (old_default_type != default_type)
throw Exception{"Cannot change column default specifier from " + toString(old_default_type) +
" to " + toString(default_type), 0 };
/// find column or throw exception
const auto find_column = [this] (NamesAndTypesList & columns) {
const auto it = std::find_if(columns.begin(), columns.end(),
std::bind(namesEqual, std::cref(column_name), std::placeholders::_1) );
if (it == columns.end())
throw Exception("Wrong column name. Cannot find column " + column_name + " to modify.",
DB::ErrorCodes::ILLEGAL_COLUMN);
return it;
};
/// find column in one of three column lists
const auto column_it = find_column(
default_type == ColumnDefaultType::Default ? columns :
default_type == ColumnDefaultType::Materialized ? materialized_columns :
alias_columns);
column_it->type = data_type;
/// remove, add or update default_expression
if (!default_expression && had_default_expr)
column_defaults.erase(column_name);
else if (default_expression && !had_default_expr)
column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression});
else
column_defaults[column_name].expression = default_expression;
}
else
throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR);
}
void AlterCommands::apply(NamesAndTypesList & columns,
NamesAndTypesList & materialized_columns,
NamesAndTypesList & alias_columns,
ColumnDefaults & column_defaults) const
{
auto new_columns = columns;
auto new_materialized_columns = materialized_columns;
auto new_alias_columns = alias_columns;
auto new_column_defaults = column_defaults;
for (const AlterCommand & command : *this)
command.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults);
columns = std::move(new_columns);
materialized_columns = std::move(new_materialized_columns);
alias_columns = std::move(new_alias_columns);
column_defaults = std::move(new_column_defaults);
}
}

2
dbms/src/Storages/ITableDeclaration.cpp
View File

@ -103,7 +103,7 @@ Block ITableDeclaration::getSampleBlockNonMaterialized() const
{
Block res;
for (const auto & col : getColumnsListAsterisk())
for (const auto & col : getColumnsListNonMaterialized())
res.insert({ col.type->createColumn(), col.type, col.name });
return res;

7
dbms/src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -360,8 +360,11 @@ void MergeTreeData::dropAllData()
void MergeTreeData::checkAlter(const AlterCommands & params)
{
/// Проверим, что указанные преобразования можно совершить над списком столбцов без учета типов.
NamesAndTypesList new_columns = *columns;
params.apply(new_columns);
auto new_columns = *columns;
auto new_materialized_columns = materialized_columns;
auto new_alias_columns = alias_columns;
auto new_column_defaults = column_defaults;
params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults);
/// Список столбцов, которые нельзя трогать.
/// sampling_expression можно не учитывать, потому что он обязан содержаться в первичном ключе.

2
dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp
View File

@ -5,6 +5,8 @@
#include <DB/DataTypes/DataTypesNumberFixed.h>
#include <DB/DataTypes/DataTypeFixedString.h>
#include <DB/DataTypes/DataTypeAggregateFunction.h>
#include <DB/DataTypes/DataTypeArray.h>
#include <DB/DataTypes/DataTypeNested.h>
#include <DB/IO/CompressedReadBuffer.h>
#include <DB/IO/HashingReadBuffer.h>
#include <DB/Columns/ColumnsNumber.h>

5
dbms/src/Storages/StorageDistributed.cpp
View File

@ -203,8 +203,9 @@ BlockOutputStreamPtr StorageDistributed::write(ASTPtr query)
void StorageDistributed::alter(const AlterCommands & params, const String & database_name, const String & table_name, Context & context)
{
auto lock = lockStructureForAlter();
params.apply(*columns);
InterpreterAlterQuery::updateMetadata(database_name, table_name, *columns, context);
params.apply(*columns, materialized_columns, alias_columns, column_defaults);
InterpreterAlterQuery::updateMetadata(database_name, table_name,
*columns, materialized_columns, alias_columns, column_defaults, context);
}
void StorageDistributed::shutdown()

5
dbms/src/Storages/StorageMerge.cpp
View File

@ -203,8 +203,9 @@ void StorageMerge::getSelectedTables(StorageVector & selected_tables) const
void StorageMerge::alter(const AlterCommands & params, const String & database_name, const String & table_name, Context & context)
{
auto lock = lockStructureForAlter();
params.apply(*columns);
InterpreterAlterQuery::updateMetadata(database_name, table_name, *columns, context);
params.apply(*columns, materialized_columns, alias_columns, column_defaults);
InterpreterAlterQuery::updateMetadata(database_name, table_name, *columns,
materialized_columns, alias_columns, column_defaults, context);
}
}

19
dbms/src/Storages/StorageMergeTree.cpp
View File

@ -129,8 +129,12 @@ void StorageMergeTree::alter(const AlterCommands & params, const String & databa
data.checkAlter(params);
NamesAndTypesList new_columns = data.getColumnsList();
params.apply(new_columns);
auto new_columns = data.getColumnsListNonMaterialized();
auto new_materialized_columns = data.materialized_columns;
auto new_alias_columns = data.alias_columns;
auto new_column_defaults = data.column_defaults;
params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults);
MergeTreeData::DataParts parts = data.getDataParts();
std::vector<MergeTreeData::AlterDataPartTransactionPtr> transactions;
@ -143,8 +147,17 @@ void StorageMergeTree::alter(const AlterCommands & params, const String & databa
auto table_hard_lock = lockStructureForAlter();
InterpreterAlterQuery::updateMetadata(database_name, table_name, new_columns, context);
InterpreterAlterQuery::updateMetadata(database_name, table_name, new_columns,
new_materialized_columns, new_alias_columns, new_column_defaults, context);
materialized_columns = new_materialized_columns;
alias_columns = new_alias_columns;
column_defaults = new_column_defaults;
data.setColumnsList(new_columns);
data.materialized_columns = std::move(new_materialized_columns);
data.alias_columns = std::move(new_alias_columns);
data.column_defaults = std::move(new_column_defaults);
for (auto & transaction : transactions)
{

44
dbms/src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -207,7 +207,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists()
zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent));
ops.push_back(new zkutil::Op::Create(zookeeper_path + "/metadata", metadata.str(),
zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent));
ops.push_back(new zkutil::Op::Create(zookeeper_path + "/columns", data.getColumnsList().toString(),
ops.push_back(new zkutil::Op::Create(zookeeper_path + "/columns", data.getColumnsListNonMaterialized().toString(),
zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent));
ops.push_back(new zkutil::Op::Create(zookeeper_path + "/log", "",
zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent));
@ -256,16 +256,23 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo
zkutil::Stat stat;
auto columns = NamesAndTypesList::parse(zookeeper->get(zookeeper_path + "/columns", &stat), context.getDataTypeFactory());
NamesAndTypesList materialized_columns;
NamesAndTypesList alias_columns;
ColumnDefaults column_defaults;
columns_version = stat.version;
if (columns != data.getColumnsList())
if (columns != data.getColumnsListNonMaterialized())
{
if (allow_alter && (data.getColumnsList().sizeOfDifference(columns) <= 2 || skip_sanity_checks))
if (allow_alter && (data.getColumnsListNonMaterialized().sizeOfDifference(columns) <= 2 || skip_sanity_checks))
{
LOG_WARNING(log, "Table structure in ZooKeeper is a little different from local table structure. Assuming ALTER.");
/// Без всяких блокировок, потому что таблица еще не создана.
InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, context);
InterpreterAlterQuery::updateMetadata(database_name, table_name, columns,
materialized_columns, alias_columns, column_defaults, context);
data.setColumnsList(columns);
data.materialized_columns = std::move(materialized_columns);
data.alias_columns = std::move(alias_columns);
data.column_defaults = std::move(column_defaults);
}
else
{
@ -403,7 +410,7 @@ void StorageReplicatedMergeTree::createReplica()
LOG_DEBUG(log, "Copied " << source_queue.size() << " queue entries");
}
zookeeper->create(replica_path + "/columns", data.getColumnsList().toString(), zkutil::CreateMode::Persistent);
zookeeper->create(replica_path + "/columns", data.getColumnsListNonMaterialized().toString(), zkutil::CreateMode::Persistent);
}
void StorageReplicatedMergeTree::activateReplica()
@ -1583,6 +1590,9 @@ void StorageReplicatedMergeTree::alterThread()
zkutil::Stat stat;
String columns_str = zookeeper->get(zookeeper_path + "/columns", &stat, alter_thread_event);
NamesAndTypesList columns = NamesAndTypesList::parse(columns_str, context.getDataTypeFactory());
NamesAndTypesList materialized_columns;
NamesAndTypesList alias_columns;
ColumnDefaults column_defaults;
bool changed_version = (stat.version != columns_version);
@ -1593,12 +1603,22 @@ void StorageReplicatedMergeTree::alterThread()
{
auto table_lock = lockStructureForAlter();
if (columns != data.getColumnsList())
if (columns != data.getColumnsListNonMaterialized())
{
LOG_INFO(log, "Columns list changed in ZooKeeper. Applying changes locally.");
InterpreterAlterQuery::updateMetadata(database_name, table_name, columns, context);
InterpreterAlterQuery::updateMetadata(database_name, table_name, columns,
materialized_columns, alias_columns, column_defaults, context);
this->materialized_columns = materialized_columns;
this->alias_columns = alias_columns;
this->column_defaults = column_defaults;
data.setColumnsList(columns);
data.materialized_columns = std::move(materialized_columns);
data.alias_columns = std::move(alias_columns);
data.column_defaults = std::move(column_defaults);
if (unreplicated_data)
unreplicated_data->setColumnsList(columns);
LOG_INFO(log, "Applied changes to table.");
@ -2306,6 +2326,9 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params,
LOG_DEBUG(log, "Doing ALTER");
NamesAndTypesList new_columns;
NamesAndTypesList new_materialized_columns;
NamesAndTypesList new_alias_columns;
ColumnDefaults new_column_defaults;
String new_columns_str;
int new_columns_version;
zkutil::Stat stat;
@ -2318,8 +2341,11 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params,
data.checkAlter(params);
new_columns = data.getColumnsList();
params.apply(new_columns);
new_columns = data.getColumnsListNonMaterialized();
new_materialized_columns = data.materialized_columns;
new_alias_columns = data.alias_columns;
new_column_defaults = data.column_defaults;
params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults);
new_columns_str = new_columns.toString();

2
dbms/src/Storages/StorageView.cpp
View File

@ -36,7 +36,7 @@ StorageView::StorageView(
const NamesAndTypesList & materialized_columns_,
const NamesAndTypesList & alias_columns_,
const ColumnDefaults & column_defaults_)
: IStorage{materialized_columns_, alias_columns_, column_defaults}, table_name(table_name_),
: IStorage{materialized_columns_, alias_columns_, column_defaults_}, table_name(table_name_),
database_name(database_name_), context(context_), columns(columns_)
{
ASTCreateQuery & create = typeid_cast<ASTCreateQuery &>(*query_);