thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

store all kinds of columns in the same list in ColumnsDescription [#CLICKHOUSE-3101]

Browse Source
This commit is contained in:
Alexey Zatelepin 2019-03-14 18:20:51 +03:00
parent d35aefefc8
commit b5a0a3fa23
34 changed files with 735 additions and 869 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dbms/programs/client/Client.cpp
View File

@ -976,7 +976,7 @@ private:
BlockInputStreamPtr block_input = context.getInputFormat(
current_format, buf, sample, insert_format_max_block_size);
const auto & column_defaults = columns_description.defaults;
const auto & column_defaults = columns_description.getDefaults();
if (!column_defaults.empty())
block_input = std::make_shared<AddingDefaultsBlockInputStream>(block_input, column_defaults, context);

3
dbms/programs/server/TCPHandler.cpp
View File

@ -723,8 +723,7 @@ bool TCPHandler::receiveData()
if (!(storage = query_context->tryGetExternalTable(external_table_name)))
{
NamesAndTypesList columns = block.getNamesAndTypesList();
storage = StorageMemory::create(external_table_name,
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}, ColumnCodecs{}});
storage = StorageMemory::create(external_table_name, ColumnsDescription{columns});
storage->startup();
query_context->addExternalTable(external_table_name, storage);
}

5
dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp
View File

@ -52,8 +52,9 @@ InputStreamFromASTInsertQuery::InputStreamFromASTInsertQuery(
res_stream = context.getInputFormat(format, *input_buffer_contacenated, header, context.getSettings().max_insert_block_size);
auto columns_description = ColumnsDescription::loadFromContext(context, ast_insert_query->database, ast_insert_query->table);
if (columns_description && !columns_description->defaults.empty())
res_stream = std::make_shared<AddingDefaultsBlockInputStream>(res_stream, columns_description->defaults, context);
auto column_defaults = columns_description->getDefaults();
if (columns_description && !column_defaults.empty())
res_stream = std::make_shared<AddingDefaultsBlockInputStream>(res_stream, column_defaults, context);
}
}

45
dbms/src/DataTypes/NestedUtils.cpp
View File

@ -77,51 +77,6 @@ std::string extractTableName(const std::string & nested_name)
}
NamesAndTypesList flatten(const NamesAndTypesList & names_and_types)
{
std::unordered_map<std::string, std::vector<std::string>> dummy;
return flattenWithMapping(names_and_types, dummy);
}
NamesAndTypesList flattenWithMapping(const NamesAndTypesList & names_and_types, std::unordered_map<std::string, std::vector<std::string>> & mapping)
{
NamesAndTypesList res;
for (const auto & name_type : names_and_types)
{
if (const DataTypeArray * type_arr = typeid_cast<const DataTypeArray *>(name_type.type.get()))
{
if (const DataTypeTuple * type_tuple = typeid_cast<const DataTypeTuple *>(type_arr->getNestedType().get()))
{
const DataTypes & elements = type_tuple->getElements();
const Strings & names = type_tuple->getElementNames();
size_t tuple_size = elements.size();
for (size_t i = 0; i < tuple_size; ++i)
{
String nested_name = concatenateName(name_type.name, names[i]);
mapping[name_type.name].push_back(nested_name);
res.emplace_back(nested_name, std::make_shared<DataTypeArray>(elements[i]));
}
}
else
{
mapping[name_type.name].push_back(name_type.name);
res.push_back(name_type);
}
}
else
{
mapping[name_type.name].push_back(name_type.name);
res.push_back(name_type);
}
}
return res;
}
Block flatten(const Block & block)
{
Block res;

3
dbms/src/DataTypes/NestedUtils.h
View File

@ -17,10 +17,7 @@ namespace Nested
std::string extractTableName(const std::string & nested_name);
/// Replace Array(Tuple(...)) columns to a multiple of Array columns in a form of `column_name.element_name`.
NamesAndTypesList flatten(const NamesAndTypesList & names_and_types);
Block flatten(const Block & block);
/// Works as normal flatten, but provide information about flattened columns
NamesAndTypesList flattenWithMapping(const NamesAndTypesList & names_and_types, std::unordered_map<std::string, std::vector<std::string>> & mapping);
/// Collect Array columns in a form of `column_name.element_name` to single Array(Tuple(...)) column.
NamesAndTypesList collect(const NamesAndTypesList & names_and_types);

322
dbms/src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -175,165 +175,6 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
}
using ColumnsAndDefaults = std::pair<NamesAndTypesList, ColumnDefaults>;
using ColumnsDeclarationAndModifiers = std::tuple<NamesAndTypesList, ColumnDefaults, ColumnCodecs, ColumnComments>;
/// AST to the list of columns with types. Columns of Nested type are expanded into a list of real columns.
static ColumnsDeclarationAndModifiers parseColumns(const ASTExpressionList & column_list_ast, const Context & context)
{
/// list of table columns in correct order
NamesAndTypesList columns{};
ColumnDefaults defaults{};
ColumnCodecs codecs{};
ColumnComments comments{};
/// Columns requiring type-deduction or default_expression type-check
std::vector<std::pair<NameAndTypePair *, ASTColumnDeclaration *>> defaulted_columns{};
/** all default_expressions as a single expression list,
* mixed with conversion-columns for each explicitly specified type */
ASTPtr default_expr_list = std::make_shared<ASTExpressionList>();
default_expr_list->children.reserve(column_list_ast.children.size());
for (const auto & ast : column_list_ast.children)
{
auto & col_decl = typeid_cast<ASTColumnDeclaration &>(*ast);
DataTypePtr column_type = nullptr;
if (col_decl.type)
{
column_type = DataTypeFactory::instance().get(col_decl.type);
columns.emplace_back(col_decl.name, column_type);
}
else
/// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions
columns.emplace_back(col_decl.name, std::make_shared<DataTypeUInt8>());
/// add column to postprocessing if there is a default_expression specified
if (col_decl.default_expression)
{
defaulted_columns.emplace_back(&columns.back(), &col_decl);
/** for columns with explicitly-specified type create two expressions:
* 1. default_expression aliased as column name with _tmp suffix
* 2. conversion of expression (1) to explicitly-specified type alias as column name */
if (col_decl.type)
{
const auto & final_column_name = col_decl.name;
const auto tmp_column_name = final_column_name + "_tmp";
const auto data_type_ptr = columns.back().type.get();
default_expr_list->children.emplace_back(setAlias(
makeASTFunction("CAST", std::make_shared<ASTIdentifier>(tmp_column_name),
std::make_shared<ASTLiteral>(data_type_ptr->getName())), final_column_name));
default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), tmp_column_name));
}
else
default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), col_decl.name));
}
if (col_decl.codec)
{
auto codec = CompressionCodecFactory::instance().get(col_decl.codec, column_type);
codecs.emplace(col_decl.name, codec);
}
if (col_decl.comment)
{
if (auto comment_str = typeid_cast<ASTLiteral &>(*col_decl.comment).value.get<String>(); !comment_str.empty())
comments.emplace(col_decl.name, comment_str);
}
}
/// set missing types and wrap default_expression's in a conversion-function if necessary
if (!defaulted_columns.empty())
{
auto syntax_analyzer_result = SyntaxAnalyzer(context).analyze(default_expr_list, columns);
const auto actions = ExpressionAnalyzer(default_expr_list, syntax_analyzer_result, context).getActions(true);
const auto block = actions->getSampleBlock();
for (auto action : actions->getActions())
if (action.type == ExpressionAction::Type::JOIN || action.type == ExpressionAction::Type::ARRAY_JOIN)
throw Exception("Cannot CREATE table. Unsupported default value that requires ARRAY JOIN or JOIN action", ErrorCodes::THERE_IS_NO_DEFAULT_VALUE);
for (auto & column : defaulted_columns)
{
const auto name_and_type_ptr = column.first;
const auto col_decl_ptr = column.second;
const auto & column_name = col_decl_ptr->name;
const auto has_explicit_type = nullptr != col_decl_ptr->type;
auto & explicit_type = name_and_type_ptr->type;
/// if column declaration contains explicit type, name_and_type_ptr->type is not null
if (has_explicit_type)
{
const auto & tmp_column = block.getByName(column_name + "_tmp");
const auto & deduced_type = tmp_column.type;
/// type mismatch between explicitly specified and deduced type, add conversion for non-array types
if (!explicit_type->equals(*deduced_type))
{
col_decl_ptr->default_expression = makeASTFunction("CAST", col_decl_ptr->default_expression,
std::make_shared<ASTLiteral>(explicit_type->getName()));
col_decl_ptr->children.clear();
col_decl_ptr->children.push_back(col_decl_ptr->type);
col_decl_ptr->children.push_back(col_decl_ptr->default_expression);
}
}
else
/// no explicit type, name_and_type_ptr->type is null, set to deduced type
explicit_type = block.getByName(column_name).type;
defaults.emplace(column_name, ColumnDefault{
columnDefaultKindFromString(col_decl_ptr->default_specifier),
col_decl_ptr->default_expression
});
}
}
std::unordered_map<std::string, std::vector<std::string>> mapping;
auto new_columns = Nested::flattenWithMapping(columns, mapping);
for (const auto & [old_name, new_names] : mapping)
{
auto codec_it = codecs.find(old_name);
if ((new_names.size() == 1 && old_name == new_names.back()) || codec_it == codecs.end())
continue;
auto codec = codec_it->second;
codecs.erase(codec_it);
for (const auto & new_name : new_names)
codecs.emplace(new_name, codec);
}
return {new_columns, defaults, codecs, comments};
}
static NamesAndTypesList removeAndReturnColumns(ColumnsAndDefaults & columns_declare, const ColumnDefaultKind kind)
{
auto & columns = std::get<0>(columns_declare);
auto & defaults = std::get<1>(columns_declare);
NamesAndTypesList removed{};
for (auto it = std::begin(columns); it != std::end(columns);)
{
const auto jt = defaults.find(it->name);
if (jt != std::end(defaults) && jt->second.kind == kind)
{
removed.push_back(*it);
it = columns.erase(it);
}
else
++it;
}
return removed;
}
ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns)
{
auto columns_list = std::make_shared<ASTExpressionList>();
@ -358,7 +199,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns)
{
auto columns_list = std::make_shared<ASTExpressionList>();
for (const auto & column : columns.getAll())
for (const auto & column : columns)
{
const auto column_declaration = std::make_shared<ASTColumnDeclaration>();
ASTPtr column_declaration_ptr{column_declaration};
@ -371,23 +212,20 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns)
const auto type_name_end = type_name_pos + type_name.size();
column_declaration->type = parseQuery(storage_p, type_name_pos, type_name_end, "data type", 0);
const auto defaults_it = columns.defaults.find(column.name);
if (defaults_it != std::end(columns.defaults))
if (column.default_desc.expression)
{
column_declaration->default_specifier = toString(defaults_it->second.kind);
column_declaration->default_expression = defaults_it->second.expression->clone();
column_declaration->default_specifier = toString(column.default_desc.kind);
column_declaration->default_expression = column.default_desc.expression->clone();
}
const auto comments_it = columns.comments.find(column.name);
if (comments_it != std::end(columns.comments))
if (!column.comment.empty())
{
column_declaration->comment = std::make_shared<ASTLiteral>(Field(comments_it->second));
column_declaration->comment = std::make_shared<ASTLiteral>(Field(column.comment));
}
const auto ct = columns.codecs.find(column.name);
if (ct != std::end(columns.codecs))
if (column.codec)
{
String codec_desc = ct->second->getCodecDesc();
String codec_desc = column.codec->getCodecDesc();
codec_desc = "CODEC(" + codec_desc + ")";
auto codec_desc_pos = codec_desc.data();
const auto codec_desc_end = codec_desc_pos + codec_desc.size();
@ -411,20 +249,114 @@ ASTPtr InterpreterCreateQuery::formatIndices(const IndicesDescription & indices)
return res;
}
ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpressionList & columns, const Context & context)
ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpressionList & columns_ast, const Context & context)
{
/// First, deduce implicit types.
/** all default_expressions as a single expression list,
* mixed with conversion-columns for each explicitly specified type */
ASTPtr default_expr_list = std::make_shared<ASTExpressionList>();
NamesAndTypesList column_names_and_types;
for (const auto & ast : columns_ast.children)
{
const auto & col_decl = typeid_cast<ASTColumnDeclaration &>(*ast);
DataTypePtr column_type = nullptr;
if (col_decl.type)
{
column_type = DataTypeFactory::instance().get(col_decl.type);
column_names_and_types.emplace_back(col_decl.name, column_type);
}
else
{
/// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions
column_names_and_types.emplace_back(col_decl.name, std::make_shared<DataTypeUInt8>());
}
/// add column to postprocessing if there is a default_expression specified
if (col_decl.default_expression)
{
/** for columns with explicitly-specified type create two expressions:
* 1. default_expression aliased as column name with _tmp suffix
* 2. conversion of expression (1) to explicitly-specified type alias as column name */
if (col_decl.type)
{
const auto & final_column_name = col_decl.name;
const auto tmp_column_name = final_column_name + "_tmp";
const auto data_type_ptr = column_names_and_types.back().type.get();
default_expr_list->children.emplace_back(setAlias(
makeASTFunction("CAST", std::make_shared<ASTIdentifier>(tmp_column_name),
std::make_shared<ASTLiteral>(data_type_ptr->getName())), final_column_name));
default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), tmp_column_name));
}
else
default_expr_list->children.emplace_back(setAlias(col_decl.default_expression->clone(), col_decl.name));
}
}
Block defaults_sample_block;
/// set missing types and wrap default_expression's in a conversion-function if necessary
if (!default_expr_list->children.empty())
{
auto syntax_analyzer_result = SyntaxAnalyzer(context).analyze(default_expr_list, column_names_and_types);
const auto actions = ExpressionAnalyzer(default_expr_list, syntax_analyzer_result, context).getActions(true);
for (auto action : actions->getActions())
if (action.type == ExpressionAction::Type::JOIN || action.type == ExpressionAction::Type::ARRAY_JOIN)
throw Exception("Cannot CREATE table. Unsupported default value that requires ARRAY JOIN or JOIN action", ErrorCodes::THERE_IS_NO_DEFAULT_VALUE);
defaults_sample_block = actions->getSampleBlock();
}
ColumnsDescription res;
auto name_type_it = column_names_and_types.begin();
for (auto ast_it = columns_ast.children.begin(); ast_it != columns_ast.children.end(); ++ast_it, ++name_type_it)
{
ColumnDescription column;
auto && parsed_columns = parseColumns(columns, context);
auto columns_and_defaults = std::make_pair(std::move(std::get<0>(parsed_columns)), std::move(std::get<1>(parsed_columns)));
res.aliases = removeAndReturnColumns(columns_and_defaults, ColumnDefaultKind::Alias);
res.materialized = removeAndReturnColumns(columns_and_defaults, ColumnDefaultKind::Materialized);
res.ordinary = std::move(columns_and_defaults.first);
res.defaults = std::move(columns_and_defaults.second);
res.codecs = std::move(std::get<2>(parsed_columns));
res.comments = std::move(std::get<3>(parsed_columns));
auto & col_decl = typeid_cast<ASTColumnDeclaration &>(*ast_it->get());
if (res.ordinary.size() + res.materialized.size() == 0)
column.name = col_decl.name;
if (col_decl.default_expression)
{
ASTPtr default_expr = col_decl.default_expression->clone();
if (col_decl.type)
{
const auto & deduced_type = defaults_sample_block.getByName(column.name + "_tmp").type;
column.type = name_type_it->type;
if (!column.type->equals(*deduced_type))
default_expr = makeASTFunction("CAST", default_expr, std::make_shared<ASTLiteral>(column.type->getName()));
}
else
column.type = defaults_sample_block.getByName(column.name).type;
column.default_desc.kind = columnDefaultKindFromString(col_decl.default_specifier);
column.default_desc.expression = default_expr;
}
else if (col_decl.type)
column.type = name_type_it->type;
else
throw Exception();
if (col_decl.comment)
{
auto comment_str = typeid_cast<ASTLiteral &>(*col_decl.comment).value.get<String>();
if (!comment_str.empty())
column.comment = comment_str;
}
if (col_decl.codec)
column.codec = CompressionCodecFactory::instance().get(col_decl.codec, column.type);
res.add(std::move(column));
}
res.flattenNested();
if (res.getAllPhysical().empty())
throw Exception{"Cannot CREATE table without physical columns", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED};
return res;
@ -434,33 +366,32 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres
ColumnsDescription InterpreterCreateQuery::setColumns(
ASTCreateQuery & create, const Block & as_select_sample, const StoragePtr & as_storage) const
{
ColumnsDescription res;
ColumnsDescription columns;
IndicesDescription indices;
if (create.columns_list)
{
if (create.columns_list->columns)
res = getColumnsDescription(*create.columns_list->columns, context);
columns = getColumnsDescription(*create.columns_list->columns, context);
if (create.columns_list->indices)
for (const auto & index : create.columns_list->indices->children)
indices.indices.push_back(
std::dynamic_pointer_cast<ASTIndexDeclaration>(index->clone()));
std::dynamic_pointer_cast<ASTIndexDeclaration>(index->clone()));
}
else if (!create.as_table.empty())
{
res = as_storage->getColumns();
columns = as_storage->getColumns();
indices = as_storage->getIndicesDescription();
}
else if (create.select)
{
for (size_t i = 0; i < as_select_sample.columns(); ++i)
res.ordinary.emplace_back(as_select_sample.safeGetByPosition(i).name, as_select_sample.safeGetByPosition(i).type);
columns = ColumnsDescription(as_select_sample.getNamesAndTypesList());
}
else
throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY);
/// Even if query has list of columns, canonicalize it (unfold Nested columns).
ASTPtr new_columns = formatColumns(res);
ASTPtr new_columns = formatColumns(columns);
ASTPtr new_indices = formatIndices(indices);
if (!create.columns_list)
@ -481,20 +412,13 @@ ColumnsDescription InterpreterCreateQuery::setColumns(
/// Check for duplicates
std::set<String> all_columns;
auto check_column_already_exists = [&all_columns](const NameAndTypePair & column_name_and_type)
for (const auto & column : columns)
{
if (!all_columns.emplace(column_name_and_type.name).second)
throw Exception("Column " + backQuoteIfNeed(column_name_and_type.name) + " already exists", ErrorCodes::DUPLICATE_COLUMN);
};
if (!all_columns.emplace(column.name).second)
throw Exception("Column " + backQuoteIfNeed(column.name) + " already exists", ErrorCodes::DUPLICATE_COLUMN);
}
for (const auto & elem : res.ordinary)
check_column_already_exists(elem);
for (const auto & elem : res.materialized)
check_column_already_exists(elem);
for (const auto & elem : res.aliases)
check_column_already_exists(elem);
return res;
return columns;
}

73
dbms/src/Interpreters/InterpreterDescribeQuery.cpp
View File

@ -58,46 +58,39 @@ Block InterpreterDescribeQuery::getSampleBlock()
BlockInputStreamPtr InterpreterDescribeQuery::executeImpl()
{
const ASTDescribeQuery & ast = typeid_cast<const ASTDescribeQuery &>(*query_ptr);
ColumnsDescription columns;
NamesAndTypesList columns;
ColumnDefaults column_defaults;
ColumnComments column_comments;
ColumnCodecs column_codecs;
StoragePtr table;
auto table_expression = typeid_cast<const ASTTableExpression *>(ast.table_expression.get());
if (table_expression->subquery)
const auto & ast = typeid_cast<const ASTDescribeQuery &>(*query_ptr);
const auto & table_expression = typeid_cast<const ASTTableExpression &>(*ast.table_expression);
if (table_expression.subquery)
{
columns = InterpreterSelectWithUnionQuery::getSampleBlock(table_expression->subquery->children[0], context).getNamesAndTypesList();
auto names_and_types = InterpreterSelectWithUnionQuery::getSampleBlock(
table_expression.subquery->children.at(0), context).getNamesAndTypesList();
columns = ColumnsDescription(std::move(names_and_types));
}
else
{
if (table_expression->table_function)
StoragePtr table;
if (table_expression.table_function)
{
auto table_function = typeid_cast<const ASTFunction *>(table_expression->table_function.get());
/// Get the table function
TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_function->name, context);
/// Run it and remember the result
table = table_function_ptr->execute(table_expression->table_function, context);
const auto & table_function = typeid_cast<const ASTFunction &>(*table_expression.table_function);
TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_function.name, context);
/// Run the table function and remember the result
table = table_function_ptr->execute(table_expression.table_function, context);
}
else
{
auto identifier = typeid_cast<const ASTIdentifier *>(table_expression->database_and_table_name.get());
const auto & identifier = typeid_cast<const ASTIdentifier &>(*table_expression.database_and_table_name);
String database_name;
String table_name;
std::tie(database_name, table_name) = IdentifierSemantic::extractDatabaseAndTable(*identifier);
std::tie(database_name, table_name) = IdentifierSemantic::extractDatabaseAndTable(identifier);
table = context.getTable(database_name, table_name);
}
auto table_lock = table->lockStructureForShare(false, context.getCurrentQueryId());
columns = table->getColumns().getAll();
column_defaults = table->getColumns().defaults;
column_comments = table->getColumns().comments;
column_codecs = table->getColumns().codecs;
columns = table->getColumns();
}
Block sample_block = getSampleBlock();
@ -108,37 +101,23 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl()
res_columns[0]->insert(column.name);
res_columns[1]->insert(column.type->getName());
const auto defaults_it = column_defaults.find(column.name);
if (defaults_it == std::end(column_defaults))
if (column.default_desc.expression)
{
res_columns[2]->insert(toString(column.default_desc.kind));
res_columns[3]->insert(queryToString(column.default_desc.expression));
}
else
{
res_columns[2]->insertDefault();
res_columns[3]->insertDefault();
}
else
{
res_columns[2]->insert(toString(defaults_it->second.kind));
res_columns[3]->insert(queryToString(defaults_it->second.expression));
}
const auto comments_it = column_comments.find(column.name);
if (comments_it == std::end(column_comments))
{
res_columns[4]->insertDefault();
}
else
{
res_columns[4]->insert(comments_it->second);
}
res_columns[4]->insert(column.comment);
const auto codecs_it = column_codecs.find(column.name);
if (codecs_it == std::end(column_codecs))
{
if (column.codec)
res_columns[5]->insert(column.codec->getCodecDesc());
else
res_columns[5]->insertDefault();
}
else
{
res_columns[5]->insert(codecs_it->second->getCodecDesc());
}
}
return std::make_shared<OneBlockInputStream>(sample_block.cloneWithColumns(std::move(res_columns)));

8
dbms/src/Interpreters/InterpreterInsertQuery.cpp
View File

@ -115,7 +115,7 @@ BlockIO InterpreterInsertQuery::execute()
/// Actually we don't know structure of input blocks from query/table,
/// because some clients break insertion protocol (columns != header)
out = std::make_shared<AddingDefaultBlockOutputStream>(
out, query_sample_block, table->getSampleBlock(), table->getColumns().defaults, context);
out, query_sample_block, table->getSampleBlock(), table->getColumns().getDefaults(), context);
auto out_wrapper = std::make_shared<CountingBlockOutputStream>(out);
out_wrapper->setProcessListElement(context.getProcessListElement());
@ -140,9 +140,9 @@ BlockIO InterpreterInsertQuery::execute()
if (!allow_materialized)
{
Block in_header = res.in->getHeader();
for (const auto & name_type : table->getColumns().materialized)
if (in_header.has(name_type.name))
throw Exception("Cannot insert column " + name_type.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN);
for (const auto & column : table->getColumns())
if (column.default_desc.kind == ColumnDefaultKind::Materialized && in_header.has(column.name))
throw Exception("Cannot insert column " + column.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN);
}
}
else if (query.data && !query.has_tail) /// can execute without additional data

16
dbms/src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -810,13 +810,13 @@ void InterpreterSelectQuery::executeFetchColumns(
/// Are ALIAS columns required for query execution?
auto alias_columns_required = false;
if (storage && !storage->getColumns().aliases.empty())
if (storage)
{
const auto & column_defaults = storage->getColumns().defaults;
for (const auto & column : required_columns)
const ColumnsDescription & storage_columns = storage->getColumns();
for (const auto & column_name : required_columns)
{
const auto default_it = column_defaults.find(column);
if (default_it != std::end(column_defaults) && default_it->second.kind == ColumnDefaultKind::Alias)
auto column_default = storage_columns.getDefault(column_name);
if (column_default && column_default->kind == ColumnDefaultKind::Alias)
{
alias_columns_required = true;
break;
@ -845,10 +845,10 @@ void InterpreterSelectQuery::executeFetchColumns(
for (const auto & column : required_columns)
{
ASTPtr column_expr;
const auto default_it = column_defaults.find(column);
bool is_alias = default_it != std::end(column_defaults) && default_it->second.kind == ColumnDefaultKind::Alias;
const auto column_default = storage_columns.getDefault(column);
bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias;
if (is_alias)
column_expr = setAlias(default_it->second.expression->clone(), column);
column_expr = setAlias(column_default->expression->clone(), column);
else
column_expr = std::make_shared<ASTIdentifier>(column);

25
dbms/src/Interpreters/MutationsInterpreter.cpp
View File

@ -126,7 +126,7 @@ static void validateUpdateColumns(
for (const String & column_name : updated_columns)
{
auto found = false;
for (const auto & col : storage->getColumns().ordinary)
for (const auto & col : storage->getColumns().getOrdinary())
{
if (col.name == column_name)
{
@ -137,7 +137,7 @@ static void validateUpdateColumns(
if (!found)
{
for (const auto & col : storage->getColumns().materialized)
for (const auto & col : storage->getColumns().getMaterialized())
{
if (col.name == column_name)
throw Exception("Cannot UPDATE materialized column `" + column_name + "`", ErrorCodes::CANNOT_UPDATE_COLUMN);
@ -187,19 +187,17 @@ void MutationsInterpreter::prepare(bool dry_run)
std::unordered_map<String, Names> column_to_affected_materialized;
if (!updated_columns.empty())
{
for (const auto & kv : columns_desc.defaults)
for (const auto & column : columns_desc)
{
const String & column = kv.first;
const ColumnDefault & col_default = kv.second;
if (col_default.kind == ColumnDefaultKind::Materialized)
if (column.default_desc.kind == ColumnDefaultKind::Materialized)
{
auto query = col_default.expression->clone();
auto query = column.default_desc.expression->clone();
auto syntax_result = SyntaxAnalyzer(context).analyze(query, all_columns);
ExpressionAnalyzer analyzer(query, syntax_result, context);
for (const String & dependency : analyzer.getRequiredSourceColumns())
{
if (updated_columns.count(dependency))
column_to_affected_materialized[dependency].push_back(column);
column_to_affected_materialized[dependency].push_back(column.name);
}
}
}
@ -250,11 +248,14 @@ void MutationsInterpreter::prepare(bool dry_run)
if (!affected_materialized.empty())
{
stages.emplace_back(context);
for (const auto & column : columns_desc.materialized)
for (const auto & column : columns_desc)
{
stages.back().column_to_updated.emplace(
column.name,
columns_desc.defaults.at(column.name).expression->clone());
if (column.default_desc.kind == ColumnDefaultKind::Materialized)
{
stages.back().column_to_updated.emplace(
column.name,
column.default_desc.expression->clone());
}
}
}
}

4
dbms/src/Interpreters/SyntaxAnalyzer.cpp
View File

@ -75,7 +75,7 @@ void collectSourceColumns(ASTSelectQuery * select_query, StoragePtr storage, Nam
if (select_query)
{
const auto & storage_aliases = storage->getColumns().aliases;
const auto & storage_aliases = storage->getColumns().getAliases();
source_columns.insert(source_columns.end(), storage_aliases.begin(), storage_aliases.end());
}
}
@ -664,7 +664,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(
}
translateQualifiedNames(query, *select_query, context,
(storage ? storage->getColumns().ordinary.getNames() : source_columns_list), source_columns_set,
(storage ? storage->getColumns().getOrdinary().getNames() : source_columns_list), source_columns_set,
result.analyzed_join.columns_from_joined_table);
/// Depending on the user's profile, check for the execution rights

4
dbms/src/Interpreters/interpretSubquery.cpp
View File

@ -64,14 +64,14 @@ std::shared_ptr<InterpreterSelectWithUnionQuery> interpretSubquery(
{
auto query_context = const_cast<Context *>(&context.getQueryContext());
const auto & storage = query_context->executeTableFunction(table_expression);
columns = storage->getColumns().ordinary;
columns = storage->getColumns().getOrdinary();
select_query->addTableFunction(*const_cast<ASTPtr *>(&table_expression));
}
else
{
DatabaseAndTableWithAlias database_table(*table);
const auto & storage = context.getTable(database_table.database, database_table.table);
columns = storage->getColumns().ordinary;
columns = storage->getColumns().getOrdinary();
select_query->replaceDatabaseAndTable(database_table.database, database_table.table);
}

322
dbms/src/Storages/AlterCommands.cpp
View File

@ -155,163 +155,46 @@ std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_
}
/// the names are the same if they match the whole name or name_without_dot matches the part of the name up to the dot
static bool namesEqual(const String & name_without_dot, const DB::NameAndTypePair & name_type)
{
String name_with_dot = name_without_dot + ".";
return (name_with_dot == name_type.name.substr(0, name_without_dot.length() + 1) || name_without_dot == name_type.name);
}
void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescription & indices_description,
ASTPtr & order_by_ast, ASTPtr & primary_key_ast) const
{
if (type == ADD_COLUMN)
{
if (columns_description.getAll().contains(column_name))
throw Exception{"Cannot add column " + column_name + ": column with this name already exists", ErrorCodes::ILLEGAL_COLUMN};
const auto add_column = [this] (NamesAndTypesList & columns)
{
auto insert_it = columns.end();
if (!after_column.empty())
{
/// We are trying to find first column from end with name `column_name` or with a name beginning with `column_name` and ".".
/// For example "fruits.bananas"
/// names are considered the same if they completely match or `name_without_dot` matches the part of the name to the point
const auto reverse_insert_it = std::find_if(columns.rbegin(), columns.rend(),
std::bind(namesEqual, std::cref(after_column), std::placeholders::_1));
if (reverse_insert_it == columns.rend())
throw Exception("Wrong column name. Cannot find column " + after_column + " to insert after",
ErrorCodes::ILLEGAL_COLUMN);
else
{
/// base returns an iterator that is already offset by one element to the right
insert_it = reverse_insert_it.base();
}
}
columns.emplace(insert_it, column_name, data_type);
};
if (default_kind == ColumnDefaultKind::Default)
add_column(columns_description.ordinary);
else if (default_kind == ColumnDefaultKind::Materialized)
add_column(columns_description.materialized);
else if (default_kind == ColumnDefaultKind::Alias)
add_column(columns_description.aliases);
else
throw Exception{"Unknown ColumnDefaultKind value", ErrorCodes::LOGICAL_ERROR};
ColumnDescription column(column_name, data_type);
if (default_expression)
columns_description.defaults.emplace(column_name, ColumnDefault{default_kind, default_expression});
{
column.default_desc.kind = default_kind;
column.default_desc.expression = default_expression;
}
column.comment = comment;
column.codec = codec;
if (codec)
columns_description.codecs.emplace(column_name, codec);
columns_description.add(column, after_column);
/// Slow, because each time a list is copied
columns_description.ordinary = Nested::flatten(columns_description.ordinary);
columns_description.flattenNested();
}
else if (type == DROP_COLUMN)
{
/// look for a column in list and remove it if present, also removing corresponding entry from column_defaults
const auto remove_column = [&columns_description, this] (NamesAndTypesList & columns)
{
auto removed = false;
NamesAndTypesList::iterator column_it;
while (columns.end() != (column_it = std::find_if(columns.begin(), columns.end(),
std::bind(namesEqual, std::cref(column_name), std::placeholders::_1))))
{
removed = true;
column_it = columns.erase(column_it);
columns_description.defaults.erase(column_name);
}
return removed;
};
if (!remove_column(columns_description.ordinary) &&
!remove_column(columns_description.materialized) &&
!remove_column(columns_description.aliases))
{
throw Exception("Wrong column name. Cannot find column " + column_name + " to drop",
ErrorCodes::ILLEGAL_COLUMN);
}
columns_description.remove(column_name);
}
else if (type == MODIFY_COLUMN)
{
ColumnDescription & column = columns_description.get(column_name);
if (codec)
columns_description.codecs[column_name] = codec;
column.codec = codec;
if (!is_mutable())
{
auto & comments = columns_description.comments;
if (comment.empty())
{
if (auto it = comments.find(column_name); it != comments.end())
comments.erase(it);
}
else
columns_description.comments[column_name] = comment;
column.comment = comment;
return;
}
const auto default_it = columns_description.defaults.find(column_name);
const auto had_default_expr = default_it != std::end(columns_description.defaults);
const auto old_default_kind = had_default_expr ? default_it->second.kind : ColumnDefaultKind{};
column.type = data_type;
/// target column list
auto & new_columns =
default_kind == ColumnDefaultKind::Default ? columns_description.ordinary
: default_kind == ColumnDefaultKind::Materialized ? columns_description.materialized
: columns_description.aliases;
/// find column or throw exception
const auto find_column = [this] (NamesAndTypesList & columns)
{
const auto it = std::find_if(columns.begin(), columns.end(),
std::bind(namesEqual, std::cref(column_name), std::placeholders::_1));
if (it == columns.end())
throw Exception("Wrong column name. Cannot find column " + column_name + " to modify",
ErrorCodes::ILLEGAL_COLUMN);
return it;
};
/// if default types differ, remove column from the old list, then add to the new list
if (default_kind != old_default_kind)
{
/// source column list
auto & old_columns =
old_default_kind == ColumnDefaultKind::Default ? columns_description.ordinary
: old_default_kind == ColumnDefaultKind::Materialized ? columns_description.materialized
: columns_description.aliases;
const auto old_column_it = find_column(old_columns);
new_columns.emplace_back(*old_column_it);
old_columns.erase(old_column_it);
/// do not forget to change the default type of old column
if (had_default_expr)
columns_description.defaults[column_name].kind = default_kind;
}
/// find column in one of three column lists
const auto column_it = find_column(new_columns);
column_it->type = data_type;
if (!default_expression && had_default_expr)
/// new column has no default expression, remove it from column_defaults along with it's type
columns_description.defaults.erase(column_name);
else if (default_expression && !had_default_expr)
/// new column has a default expression while the old one had not, add it it column_defaults
columns_description.defaults.emplace(column_name, ColumnDefault{default_kind, default_expression});
else if (had_default_expr)
/// both old and new columns have default expression, update it
columns_description.defaults[column_name].expression = default_expression;
column.default_desc.kind = default_kind;
column.default_desc.expression = default_expression;
}
else if (type == MODIFY_ORDER_BY)
{
@ -326,7 +209,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri
}
else if (type == COMMENT_COLUMN)
{
columns_description.comments[column_name] = comment;
columns_description.get(column_name).comment = comment;
}
else if (type == ADD_INDEX)
{
@ -415,25 +298,26 @@ void AlterCommands::apply(ColumnsDescription & columns_description, IndicesDescr
void AlterCommands::validate(const IStorage & table, const Context & context)
{
auto all_columns = table.getColumns().getAll();
auto defaults = table.getColumns().defaults;
std::vector<std::pair<NameAndTypePair, AlterCommand *>> defaulted_columns{};
/// A temporary object that is used to keep track of the current state of columns after applying a subset of commands.
auto columns = table.getColumns();
/// Default expressions will be added to this list for type deduction.
auto default_expr_list = std::make_shared<ASTExpressionList>();
default_expr_list->children.reserve(defaults.size());
/// We will save ALTER ADD/MODIFY command indices (only the last for each column) for possible modification
/// (we might need to add deduced types or modify default expressions).
/// Saving indices because we can add new commands later and thus cause vector resize.
std::unordered_map<String, size_t> column_to_command_idx;
for (AlterCommand & command : *this)
for (size_t i = 0; i < size(); ++i)
{
auto & command = (*this)[i];
if (command.type == AlterCommand::ADD_COLUMN || command.type == AlterCommand::MODIFY_COLUMN)
{
const auto & column_name = command.column_name;
const auto column_it = std::find_if(std::begin(all_columns), std::end(all_columns),
std::bind(namesEqual, std::cref(command.column_name), std::placeholders::_1));
if (command.type == AlterCommand::ADD_COLUMN)
{
if (std::end(all_columns) != column_it)
if (columns.has(column_name) || columns.hasNested(column_name))
{
if (command.if_not_exists)
command.ignore = true;
@ -443,8 +327,7 @@ void AlterCommands::validate(const IStorage & table, const Context & context)
}
else if (command.type == AlterCommand::MODIFY_COLUMN)
{
if (std::end(all_columns) == column_it)
if (!columns.has(column_name))
{
if (command.if_exists)
command.ignore = true;
@ -453,95 +336,72 @@ void AlterCommands::validate(const IStorage & table, const Context & context)
}
if (!command.ignore)
{
all_columns.erase(column_it);
defaults.erase(column_name);
}
columns.remove(column_name);
}
if (!command.ignore)
{
column_to_command_idx[column_name] = i;
/// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions
all_columns.emplace_back(column_name, command.data_type ? command.data_type : std::make_shared<DataTypeUInt8>());
columns.add(ColumnDescription(
column_name, command.data_type ? command.data_type : std::make_shared<DataTypeUInt8>()));
if (command.default_expression)
{
if (command.data_type)
{
const auto &final_column_name = column_name;
const auto & final_column_name = column_name;
const auto tmp_column_name = final_column_name + "_tmp";
const auto column_type_raw_ptr = command.data_type.get();
default_expr_list->children.emplace_back(setAlias(
makeASTFunction("CAST", std::make_shared<ASTIdentifier>(tmp_column_name),
std::make_shared<ASTLiteral>(column_type_raw_ptr->getName())),
std::make_shared<ASTLiteral>(command.data_type->getName())),
final_column_name));
default_expr_list->children.emplace_back(setAlias(command.default_expression->clone(), tmp_column_name));
defaulted_columns.emplace_back(NameAndTypePair{column_name, command.data_type}, &command);
}
else
{
/// no type explicitly specified, will deduce later
default_expr_list->children.emplace_back(
setAlias(command.default_expression->clone(), column_name));
defaulted_columns.emplace_back(NameAndTypePair{column_name, nullptr}, &command);
}
}
}
}
else if (command.type == AlterCommand::DROP_COLUMN)
{
for (const auto & default_column : defaults)
if (columns.has(command.column_name) || columns.hasNested(command.column_name))
{
const auto & default_expression = default_column.second.expression;
ASTPtr query = default_expression;
auto syntax_result = SyntaxAnalyzer(context).analyze(query, all_columns);
const auto actions = ExpressionAnalyzer(query, syntax_result, context).getActions(true);
const auto required_columns = actions->getRequiredColumns();
if (required_columns.end() != std::find(required_columns.begin(), required_columns.end(), command.column_name))
throw Exception(
"Cannot drop column " + command.column_name + ", because column " + default_column.first +
" depends on it", ErrorCodes::ILLEGAL_COLUMN);
}
auto found = false;
for (auto it = std::begin(all_columns); it != std::end(all_columns);)
{
if (namesEqual(command.column_name, *it))
for (const ColumnDescription & column : columns)
{
found = true;
it = all_columns.erase(it);
const auto & default_expression = column.default_desc.expression;
if (!default_expression)
continue;
ASTPtr query = default_expression->clone();
auto syntax_result = SyntaxAnalyzer(context).analyze(query, columns.getAll());
const auto actions = ExpressionAnalyzer(query, syntax_result, context).getActions(true);
const auto required_columns = actions->getRequiredColumns();
if (required_columns.end() != std::find(required_columns.begin(), required_columns.end(), command.column_name))
throw Exception(
"Cannot drop column " + command.column_name + ", because column " + column.name +
" depends on it", ErrorCodes::ILLEGAL_COLUMN);
}
else
++it;
}
for (auto it = std::begin(defaults); it != std::end(defaults);)
{
if (namesEqual(command.column_name, { it->first, nullptr }))
it = defaults.erase(it);
else
++it;
}
if (!found)
{
if (command.if_exists)
command.ignore = true;
else
throw Exception("Wrong column name. Cannot find column " + command.column_name + " to drop",
ErrorCodes::ILLEGAL_COLUMN);
columns.remove(command.column_name);
}
else if (command.if_exists)
command.ignore = true;
else
throw Exception("Wrong column name. Cannot find column " + command.column_name + " to drop",
ErrorCodes::ILLEGAL_COLUMN);
}
else if (command.type == AlterCommand::COMMENT_COLUMN)
{
const auto column_it = std::find_if(std::begin(all_columns), std::end(all_columns),
std::bind(namesEqual, std::cref(command.column_name), std::placeholders::_1));
if (column_it == std::end(all_columns))
if (!columns.has(command.column_name))
{
if (command.if_exists)
command.ignore = true;
@ -552,72 +412,64 @@ void AlterCommands::validate(const IStorage & table, const Context & context)
}
/** Existing defaulted columns may require default expression extensions with a type conversion,
* therefore we add them to defaulted_columns to allow further processing */
for (const auto & col_def : defaults)
* therefore we add them to default_expr_list to recalculate their types */
for (const auto & column : columns)
{
const auto & column_name = col_def.first;
const auto column_it = std::find_if(all_columns.begin(), all_columns.end(), [&] (const NameAndTypePair & name_type)
{ return namesEqual(column_name, name_type); });
const auto tmp_column_name = column_name + "_tmp";
const auto & column_type_ptr = column_it->type;
if (column.default_desc.expression)
{
const auto tmp_column_name = column.name + "_tmp";
default_expr_list->children.emplace_back(setAlias(
makeASTFunction("CAST", std::make_shared<ASTIdentifier>(tmp_column_name),
std::make_shared<ASTLiteral>(column_type_ptr->getName())),
column_name));
makeASTFunction("CAST", std::make_shared<ASTIdentifier>(tmp_column_name),
std::make_shared<ASTLiteral>(column.type->getName())),
column.name));
default_expr_list->children.emplace_back(setAlias(col_def.second.expression->clone(), tmp_column_name));
defaulted_columns.emplace_back(NameAndTypePair{column_name, column_type_ptr}, nullptr);
default_expr_list->children.emplace_back(setAlias(column.default_desc.expression->clone(), tmp_column_name));
}
}
ASTPtr query = default_expr_list;
auto syntax_result = SyntaxAnalyzer(context).analyze(query, all_columns);
auto syntax_result = SyntaxAnalyzer(context).analyze(query, columns.getAll());
const auto actions = ExpressionAnalyzer(query, syntax_result, context).getActions(true);
const auto block = actions->getSampleBlock();
/// set deduced types, modify default expression if necessary
for (auto & defaulted_column : defaulted_columns)
for (const auto & column : columns)
{
const auto & name_and_type = defaulted_column.first;
AlterCommand * & command_ptr = defaulted_column.second;
AlterCommand * command = nullptr;
auto command_it = column_to_command_idx.find(column.name);
if (command_it != column_to_command_idx.end())
command = &(*this)[command_it->second];
const auto & column_name = name_and_type.name;
const auto has_explicit_type = nullptr != name_and_type.type;
if (!(command && command->default_expression) && !column.default_desc.expression)
continue;
/// default expression on old column
if (has_explicit_type)
const DataTypePtr & explicit_type = command ? command->data_type : column.type;
if (explicit_type)
{
const auto & explicit_type = name_and_type.type;
const auto & tmp_column = block.getByName(column_name + "_tmp");
const auto & tmp_column = block.getByName(column.name + "_tmp");
const auto & deduced_type = tmp_column.type;
// column not specified explicitly in the ALTER query may require default_expression modification
if (!explicit_type->equals(*deduced_type))
{
const auto default_it = defaults.find(column_name);
/// column has no associated alter command, let's create it
if (!command_ptr)
if (!command)
{
/// column has no associated alter command, let's create it
/// add a new alter command to modify existing column
this->emplace_back(AlterCommand{
AlterCommand::MODIFY_COLUMN, column_name, explicit_type,
default_it->second.kind, default_it->second.expression
});
this->emplace_back(AlterCommand{AlterCommand::MODIFY_COLUMN,
column.name, explicit_type, column.default_desc.kind, column.default_desc.expression});
command_ptr = &this->back();
command = &back();
}
command_ptr->default_expression = makeASTFunction("CAST", command_ptr->default_expression->clone(),
command->default_expression = makeASTFunction("CAST",
command->default_expression->clone(),
std::make_shared<ASTLiteral>(explicit_type->getName()));
}
}
else
{
/// just set deduced type
command_ptr->data_type = block.getByName(column_name).type;
command->data_type = block.getByName(column.name).type;
}
}
}

3
dbms/src/Storages/ColumnDefault.cpp
View File

@ -50,7 +50,8 @@ std::string toString(const ColumnDefaultKind kind)
bool operator==(const ColumnDefault & lhs, const ColumnDefault & rhs)
{
return lhs.kind == rhs.kind && queryToString(lhs.expression) == queryToString(rhs.expression);
auto expression_str = [](const ASTPtr & expr) { return expr ? queryToString(expr) : String(); };
return lhs.kind == rhs.kind && expression_str(lhs.expression) == expression_str(rhs.expression);
}
}

2
dbms/src/Storages/ColumnDefault.h
View File

@ -23,7 +23,7 @@ std::string toString(const ColumnDefaultKind type);
struct ColumnDefault
{
ColumnDefaultKind kind;
ColumnDefaultKind kind = ColumnDefaultKind::Default;
ASTPtr expression;
};

454
dbms/src/Storages/ColumnsDescription.cpp
View File

@ -9,19 +9,16 @@
#include <IO/WriteHelpers.h>
#include <IO/ReadBuffer.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/ReadBufferFromString.h>
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/NestedUtils.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeTuple.h>
#include <Common/Exception.h>
#include <Interpreters/Context.h>
#include <Storages/IStorage.h>
#include <Common/typeid_cast.h>
#include <ext/collection_cast.h>
#include <ext/map.h>
#include <boost/range/join.hpp>
#include <Compression/CompressionFactory.h>
#include <optional>
@ -33,126 +30,54 @@ namespace DB
namespace ErrorCodes
{
extern const int NO_SUCH_COLUMN_IN_TABLE;
extern const int ILLEGAL_COLUMN;
extern const int CANNOT_PARSE_TEXT;
}
NamesAndTypesList ColumnsDescription::getAllPhysical() const
bool ColumnDescription::operator==(const ColumnDescription & other) const
{
return ext::collection_cast<NamesAndTypesList>(boost::join(ordinary, materialized));
auto codec_str = [](const CompressionCodecPtr & codec) { return codec ? codec->getCodecDesc() : String(); };
return name == other.name
&& type->equals(*other.type)
&& default_desc == other.default_desc
&& comment == other.comment
&& codec_str(codec) == codec_str(other.codec);
}
NamesAndTypesList ColumnsDescription::getAll() const
void ColumnDescription::writeText(WriteBuffer & buf) const
{
return ext::collection_cast<NamesAndTypesList>(boost::join(ordinary, boost::join(materialized, aliases)));
}
writeBackQuotedString(name, buf);
writeChar(' ', buf);
DB::writeText(type->getName(), buf);
Names ColumnsDescription::getNamesOfPhysical() const
{
return ext::map<Names>(boost::join(ordinary, materialized), [] (const auto & it) { return it.name; });
}
NameAndTypePair ColumnsDescription::getPhysical(const String & column_name) const
{
for (auto & it : boost::join(ordinary, materialized))
if (it.name == column_name)
return it;
throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
}
bool ColumnsDescription::hasPhysical(const String & column_name) const
{
for (auto & it : boost::join(ordinary, materialized))
if (it.name == column_name)
return true;
return false;
}
bool ColumnsDescription::operator==(const ColumnsDescription & other) const
{
if (ordinary != other.ordinary
|| materialized != other.materialized
|| aliases != other.aliases
|| defaults != other.defaults
|| comments != other.comments)
if (default_desc.expression)
{
return false;
writeChar('\t', buf);
DB::writeText(DB::toString(default_desc.kind), buf);
writeChar('\t', buf);
DB::writeText(queryToString(default_desc.expression), buf);
}
if (codecs.size() != other.codecs.size())
return false;
for (const auto & [col_name, codec_ptr] : codecs)
if (!comment.empty())
{
if (other.codecs.count(col_name) == 0)
return false;
if (other.codecs.at(col_name)->getCodecDesc() != codec_ptr->getCodecDesc())
return false;
writeChar('\t', buf);
DB::writeText("COMMENT ", buf);
DB::writeText(queryToString(ASTLiteral(Field(comment))), buf);
}
return true;
}
String ColumnsDescription::toString() const
{
WriteBufferFromOwnString buf;
writeCString("columns format version: 1\n", buf);
writeText(ordinary.size() + materialized.size() + aliases.size(), buf);
writeCString(" columns:\n", buf);
const auto write_columns = [this, &buf] (const NamesAndTypesList & columns)
if (codec)
{
for (const auto & column : columns)
{
const auto defaults_it = defaults.find(column.name);
const auto comments_it = comments.find(column.name);
const auto codec_it = codecs.find(column.name);
writeChar('\t', buf);
DB::writeText("CODEC(", buf);
DB::writeText(codec->getCodecDesc(), buf);
DB::writeText(")", buf);
}
writeBackQuotedString(column.name, buf);
writeChar(' ', buf);
writeText(column.type->getName(), buf);
const bool exist_comment = comments_it != std::end(comments);
const bool exist_codec = codec_it != std::end(codecs);
if (defaults_it != std::end(defaults))
{
writeChar('\t', buf);
writeText(DB::toString(defaults_it->second.kind), buf);
writeChar('\t', buf);
writeText(queryToString(defaults_it->second.expression), buf);
}
if (exist_comment)
{
writeChar('\t', buf);
writeText("COMMENT ", buf);
writeText(queryToString(ASTLiteral(Field(comments_it->second))), buf);
}
if (exist_codec)
{
writeChar('\t', buf);
writeText("CODEC(", buf);
writeText(codec_it->second->getCodecDesc(), buf);
writeText(")", buf);
}
writeChar('\n', buf);
}
};
write_columns(ordinary);
write_columns(materialized);
write_columns(aliases);
return buf.str();
writeChar('\n', buf);
}
void parseColumn(ReadBufferFromString & buf, ColumnsDescription & result, const DataTypeFactory & data_type_factory)
void ColumnDescription::readText(ReadBuffer & buf)
{
ParserColumnDeclaration column_parser(true);
String column_line;
@ -160,60 +85,303 @@ void parseColumn(ReadBufferFromString & buf, ColumnsDescription & result, const
ASTPtr ast = parseQuery(column_parser, column_line, "column parser", 0);
if (const ASTColumnDeclaration * col_ast = typeid_cast<const ASTColumnDeclaration *>(ast.get()))
{
String column_name = col_ast->name;
auto type = data_type_factory.get(col_ast->type);
name = col_ast->name;
type = DataTypeFactory::instance().get(col_ast->type);
if (col_ast->default_expression)
{
auto kind = columnDefaultKindFromString(col_ast->default_specifier);
switch (kind)
{
case ColumnDefaultKind::Default:
result.ordinary.emplace_back(column_name, std::move(type));
break;
case ColumnDefaultKind::Materialized:
result.materialized.emplace_back(column_name, std::move(type));
break;
case ColumnDefaultKind::Alias:
result.aliases.emplace_back(column_name, std::move(type));
break;
}
result.defaults.emplace(column_name, ColumnDefault{kind, std::move(col_ast->default_expression)});
default_desc.kind = columnDefaultKindFromString(col_ast->default_specifier);
default_desc.expression = std::move(col_ast->default_expression);
}
else
result.ordinary.emplace_back(column_name, std::move(type));
if (col_ast->comment)
if (auto comment_str = typeid_cast<ASTLiteral &>(*col_ast->comment).value.get<String>(); !comment_str.empty())
result.comments.emplace(column_name, std::move(comment_str));
comment = typeid_cast<ASTLiteral &>(*col_ast->comment).value.get<String>();
if (col_ast->codec)
{
auto codec = CompressionCodecFactory::instance().get(col_ast->codec, type);
result.codecs.emplace(column_name, std::move(codec));
}
codec = CompressionCodecFactory::instance().get(col_ast->codec, type);
}
else
throw Exception("Cannot parse column description", ErrorCodes::CANNOT_PARSE_TEXT);
}
CompressionCodecPtr ColumnsDescription::getCodecOrDefault(const String & column_name, CompressionCodecPtr default_codec) const
ColumnsDescription::ColumnsDescription(NamesAndTypesList ordinary)
{
const auto codec = codecs.find(column_name);
if (codec == codecs.end())
return default_codec;
return codec->second;
for (auto & elem : ordinary)
add(ColumnDescription(std::move(elem.name), std::move(elem.type)));
}
/// We are trying to find first column from end with name `column_name` or with a name beginning with `column_name` and ".".
/// For example "fruits.bananas"
/// names are considered the same if they completely match or `name_without_dot` matches the part of the name to the point
static auto getNameRange(const std::list<ColumnDescription> & columns, const String & name_without_dot)
{
String name_with_dot = name_without_dot + ".";
auto begin = columns.begin();
for (; begin != columns.end(); ++begin)
{
if (begin->name == name_without_dot)
return std::make_pair(begin, std::next(begin));
if (startsWith(begin->name, name_with_dot))
break;
}
if (begin == columns.end())
return std::make_pair(begin, begin);
auto end = std::next(begin);
for (; end != columns.end(); ++end)
{
if (!startsWith(end->name, name_with_dot))
break;
}
return std::make_pair(begin, end);
}
void ColumnsDescription::add(ColumnDescription column, const String & after_column)
{
if (has(column.name))
throw Exception("Cannot add column " + column.name + ": column with this name already exists",
ErrorCodes::ILLEGAL_COLUMN);
auto insert_it = columns.cend();
if (!after_column.empty())
{
auto range = getNameRange(columns, after_column);
if (range.first == range.second)
throw Exception("Wrong column name. Cannot find column " + after_column + " to insert after",
ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
insert_it = range.second;
}
auto it = columns.insert(insert_it, std::move(column));
name_to_column.emplace(it->name, it);
}
void ColumnsDescription::remove(const String & column_name)
{
auto range = getNameRange(columns, column_name);
if (range.first == range.second)
throw Exception("There is no column " + column_name + " in table.",
ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
for (auto list_it = range.first; list_it != range.second;)
{
name_to_column.erase(list_it->name);
list_it = columns.erase(list_it);
}
}
void ColumnsDescription::flattenNested()
{
for (auto it = columns.begin(); it != columns.end(); ++it)
{
if (const DataTypeArray * type_arr = typeid_cast<const DataTypeArray *>(it->type.get()))
{
if (const DataTypeTuple * type_tuple = typeid_cast<const DataTypeTuple *>(type_arr->getNestedType().get()))
{
ColumnDescription column = std::move(*it);
it = columns.erase(it);
name_to_column.erase(column.name);
const DataTypes & elements = type_tuple->getElements();
const Strings & names = type_tuple->getElementNames();
size_t tuple_size = elements.size();
for (size_t i = 0; i < tuple_size; ++i)
{
auto nested_column = column;
/// TODO: what to do with default expressions?
nested_column.name = Nested::concatenateName(column.name, names[i]);
nested_column.type = std::make_shared<DataTypeArray>(elements[i]);
auto inserted_it = columns.insert(it, std::move(nested_column));
name_to_column.emplace(inserted_it->name, inserted_it);
}
}
}
}
}
NamesAndTypesList ColumnsDescription::getOrdinary() const
{
NamesAndTypesList ret;
for (const auto & col : columns)
{
if (col.default_desc.kind == ColumnDefaultKind::Default)
ret.emplace_back(col.name, col.type);
}
return ret;
}
NamesAndTypesList ColumnsDescription::getMaterialized() const
{
NamesAndTypesList ret;
for (const auto & col : columns)
{
if (col.default_desc.kind == ColumnDefaultKind::Materialized)
ret.emplace_back(col.name, col.type);
}
return ret;
}
NamesAndTypesList ColumnsDescription::getAliases() const
{
NamesAndTypesList ret;
for (const auto & col : columns)
{
if (col.default_desc.kind == ColumnDefaultKind::Alias)
ret.emplace_back(col.name, col.type);
}
return ret;
}
NamesAndTypesList ColumnsDescription::getAll() const
{
NamesAndTypesList ret;
for (const auto & col : columns)
ret.emplace_back(col.name, col.type);
return ret;
}
bool ColumnsDescription::has(const String & column_name) const
{
auto it = name_to_column.find(column_name);
return it != name_to_column.end();
}
bool ColumnsDescription::hasNested(const String & column_name) const
{
auto range = getNameRange(columns, column_name);
return range.first != range.second && range.first->name.length() > column_name.length();
}
ColumnDescription & ColumnsDescription::get(const String & column_name)
{
auto it = name_to_column.find(column_name);
if (it == name_to_column.end())
throw Exception("There is no column " + column_name + " in table.",
ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
return *it->second;
}
const ColumnDescription & ColumnsDescription::get(const String & column_name) const
{
auto it = name_to_column.find(column_name);
if (it == name_to_column.end())
throw Exception("There is no column " + column_name + " in table.",
ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
return *it->second;
}
NamesAndTypesList ColumnsDescription::getAllPhysical() const
{
NamesAndTypesList ret;
for (const auto & col : columns)
{
if (col.default_desc.kind != ColumnDefaultKind::Alias)
ret.emplace_back(col.name, col.type);
}
return ret;
}
Names ColumnsDescription::getNamesOfPhysical() const
{
Names ret;
for (const auto & col : columns)
{
if (col.default_desc.kind != ColumnDefaultKind::Alias)
ret.emplace_back(col.name);
}
return ret;
}
NameAndTypePair ColumnsDescription::getPhysical(const String & column_name) const
{
auto it = name_to_column.find(column_name);
if (it == name_to_column.end() || it->second->default_desc.kind == ColumnDefaultKind::Alias)
throw Exception("There is no physical column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
return NameAndTypePair(it->second->name, it->second->type);
}
bool ColumnsDescription::hasPhysical(const String & column_name) const
{
auto it = name_to_column.find(column_name);
return it != name_to_column.end() && it->second->default_desc.kind != ColumnDefaultKind::Alias;
}
ColumnDefaults ColumnsDescription::getDefaults() const
{
ColumnDefaults ret;
for (const auto & column : columns)
{
if (column.default_desc.expression)
ret.emplace(column.name, column.default_desc);
}
return ret;
}
bool ColumnsDescription::hasDefault(const String & column_name) const
{
auto it = name_to_column.find(column_name);
return it != name_to_column.end() && it->second->default_desc.expression;
}
std::optional<ColumnDefault> ColumnsDescription::getDefault(const String & column_name) const
{
auto it = name_to_column.find(column_name);
if (it != name_to_column.end() && it->second->default_desc.expression)
return it->second->default_desc;
return {};
}
CompressionCodecPtr ColumnsDescription::getCodecOrDefault(const String & column_name, CompressionCodecPtr default_codec) const
{
const auto it = name_to_column.find(column_name);
if (it == name_to_column.end() || !it->second->codec)
return default_codec;
return it->second->codec;
}
CompressionCodecPtr ColumnsDescription::getCodecOrDefault(const String & column_name) const
{
return getCodecOrDefault(column_name, CompressionCodecFactory::instance().getDefaultCodec());
}
String ColumnsDescription::toString() const
{
WriteBufferFromOwnString buf;
writeCString("columns format version: 1\n", buf);
DB::writeText(columns.size(), buf);
writeCString(" columns:\n", buf);
for (const ColumnDescription & column : columns)
column.writeText(buf);
return buf.str();
}
ColumnsDescription ColumnsDescription::parse(const String & str)
{
ReadBufferFromString buf{str};
@ -223,13 +391,13 @@ ColumnsDescription ColumnsDescription::parse(const String & str)
readText(count, buf);
assertString(" columns:\n", buf);
const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
ColumnsDescription result;
for (size_t i = 0; i < count; ++i)
{
parseColumn(buf, result, data_type_factory);
ColumnDescription column;
column.readText(buf);
buf.ignore(1); /// ignore new line
result.add(std::move(column));
}
assertEOF(buf);

78
dbms/src/Storages/ColumnsDescription.h
View File

@ -5,64 +5,78 @@
#include <Storages/ColumnDefault.h>
#include <Core/Block.h>
#include <Storages/ColumnCodec.h>
#include <optional>
namespace DB
{
/// key-values column_name, column_comment. column_comment should be non empty.
using ColumnComments = std::unordered_map<std::string, String>;
struct ColumnDescription
{
String name;
DataTypePtr type;
ColumnDefault default_desc;
String comment;
CompressionCodecPtr codec;
ColumnDescription() = default;
ColumnDescription(String name_, DataTypePtr type_) : name(std::move(name_)), type(std::move(type_)) {}
bool operator==(const ColumnDescription & other) const;
bool operator!=(const ColumnDescription & other) const { return !(*this == other); }
void writeText(WriteBuffer & buf) const;
void readText(ReadBuffer & buf);
};
struct ColumnsDescription
{
NamesAndTypesList ordinary;
NamesAndTypesList materialized;
NamesAndTypesList aliases;
ColumnDefaults defaults;
ColumnComments comments;
ColumnCodecs codecs;
private:
std::list<ColumnDescription> columns;
std::unordered_map<String, std::list<ColumnDescription>::iterator> name_to_column;
public:
ColumnsDescription() = default;
explicit ColumnsDescription(NamesAndTypesList ordinary_);
ColumnsDescription(
NamesAndTypesList ordinary_,
NamesAndTypesList materialized_,
NamesAndTypesList aliases_,
ColumnDefaults defaults_,
ColumnComments comments_,
ColumnCodecs codecs_)
: ordinary(std::move(ordinary_))
, materialized(std::move(materialized_))
, aliases(std::move(aliases_))
, defaults(std::move(defaults_))
, comments(std::move(comments_))
, codecs(std::move(codecs_))
{}
/// `after_column` can be a Nested column name;
void add(ColumnDescription column, const String & after_column = String());
/// `column_name` can be a Nested column name;
void remove(const String & column_name);
explicit ColumnsDescription(NamesAndTypesList ordinary_) : ordinary(std::move(ordinary_)) {}
bool operator==(const ColumnsDescription & other) const;
void flattenNested(); /// TODO: remove, insert already flattened Nested columns.
bool operator==(const ColumnsDescription & other) const { return columns == other.columns; }
bool operator!=(const ColumnsDescription & other) const { return !(*this == other); }
/// ordinary + materialized.
NamesAndTypesList getAllPhysical() const;
std::list<ColumnDescription>::const_iterator begin() const { return columns.begin(); }
std::list<ColumnDescription>::const_iterator end() const { return columns.end(); }
NamesAndTypesList getOrdinary() const;
NamesAndTypesList getMaterialized() const;
NamesAndTypesList getAliases() const;
/// ordinary + materialized + aliases.
NamesAndTypesList getAll() const;
Names getNamesOfPhysical() const;
bool has(const String & column_name) const;
bool hasNested(const String & column_name) const;
ColumnDescription & get(const String & column_name);
const ColumnDescription & get(const String & column_name) const;
/// ordinary + materialized.
NamesAndTypesList getAllPhysical() const;
Names getNamesOfPhysical() const;
bool hasPhysical(const String & column_name) const;
NameAndTypePair getPhysical(const String & column_name) const;
bool hasPhysical(const String & column_name) const;
String toString() const;
ColumnDefaults getDefaults() const; /// TODO: remove
bool hasDefault(const String & column_name) const;
std::optional<ColumnDefault> getDefault(const String & column_name) const;
CompressionCodecPtr getCodecOrDefault(const String & column_name, CompressionCodecPtr default_codec) const;
CompressionCodecPtr getCodecOrDefault(const String & column_name) const;
String toString() const;
static ColumnsDescription parse(const String & str);
static const ColumnsDescription * loadFromContext(const Context & context, const String & db, const String & table);

4
dbms/src/Storages/ITableDeclaration.cpp
View File

@ -26,7 +26,7 @@ namespace ErrorCodes
void ITableDeclaration::setColumns(ColumnsDescription columns_)
{
if (columns_.ordinary.empty())
if (columns_.getOrdinary().empty())
throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
columns = std::move(columns_);
}
@ -63,7 +63,7 @@ Block ITableDeclaration::getSampleBlockNonMaterialized() const
{
Block res;
for (const auto & col : getColumns().ordinary)
for (const auto & col : getColumns().getOrdinary())
res.insert({ col.type->createColumn(), col.type, col.name });
return res;

7
dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
View File

@ -26,14 +26,13 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const MergeTreeData
continue;
}
const auto default_it = storage.getColumns().defaults.find(column_name);
/// columns has no explicit default expression
if (default_it == std::end(storage.getColumns().defaults))
const auto column_default = storage.getColumns().getDefault(column_name);
if (!column_default)
continue;
/// collect identifiers required for evaluation
IdentifierNameSet identifiers;
default_it->second.expression->collectIdentifierNames(identifiers);
column_default->expression->collectIdentifierNames(identifiers);
for (const auto & identifier : identifiers)
{

2
dbms/src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -301,7 +301,7 @@ void MergeTreeData::setPrimaryKeyIndicesAndColumns(
"added to the sorting key. You can add expressions that use only the newly added columns",
ErrorCodes::BAD_ARGUMENTS);
if (new_columns.defaults.count(col))
if (new_columns.getDefaults().count(col))
throw Exception("Newly added column " + col + " has a default expression, so adding "
"expressions that use it to the sorting key is forbidden",
ErrorCodes::BAD_ARGUMENTS);

4
dbms/src/Storages/MergeTree/MergeTreeReader.cpp
View File

@ -300,7 +300,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, bool & should_reorder, boo
if (!has_column)
{
should_reorder = true;
if (storage.getColumns().defaults.count(requested_column.name) != 0)
if (storage.getColumns().hasDefault(requested_column.name))
{
should_evaluate_missing_defaults = true;
continue;
@ -367,7 +367,7 @@ void MergeTreeReader::evaluateMissingDefaults(Block & res)
{
try
{
DB::evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.global_context);
DB::evaluateMissingDefaults(res, columns, storage.getColumns().getDefaults(), storage.global_context);
}
catch (Exception & e)
{

4
dbms/src/Storages/StorageBuffer.cpp
View File

@ -200,9 +200,9 @@ BlockInputStreams StorageBuffer::read(
for (auto & stream : streams_from_dst)
{
stream = std::make_shared<AddingMissedBlockInputStream>(
stream, header_after_adding_defaults, getColumns().defaults, context);
stream, header_after_adding_defaults, getColumns().getDefaults(), context);
stream = std::make_shared<ConvertingBlockInputStream>(
context, stream, header, ConvertingBlockInputStream::MatchColumnsMode::Name);
context, stream, header, ConvertingBlockInputStream::MatchColumnsMode::Name);
}
}
}

56
dbms/src/Storages/StorageCatBoostPool.cpp
View File

@ -225,35 +225,57 @@ void StorageCatBoostPool::createSampleBlockAndColumns()
ColumnsDescription columns;
NamesAndTypesList cat_columns;
NamesAndTypesList num_columns;
NamesAndTypesList other_columns;
sample_block.clear();
auto get_type = [](DatasetColumnType column_type) -> DataTypePtr
{
if (column_type == DatasetColumnType::Categ
|| column_type == DatasetColumnType::Auxiliary
|| column_type == DatasetColumnType::DocId)
return std::make_shared<DataTypeString>();
else
return std::make_shared<DataTypeFloat64>();
};
for (auto & desc : columns_description)
{
DataTypePtr type;
if (desc.column_type == DatasetColumnType::Categ
|| desc.column_type == DatasetColumnType::Auxiliary
|| desc.column_type == DatasetColumnType::DocId)
type = std::make_shared<DataTypeString>();
else
type = std::make_shared<DataTypeFloat64>();
DataTypePtr type = get_type(desc.column_type);
if (desc.column_type == DatasetColumnType::Categ)
cat_columns.emplace_back(desc.column_name, type);
else if (desc.column_type == DatasetColumnType::Num)
num_columns.emplace_back(desc.column_name, type);
else
columns.materialized.emplace_back(desc.column_name, type);
if (!desc.alias.empty())
{
auto alias = std::make_shared<ASTIdentifier>(desc.column_name);
columns.defaults[desc.alias] = {ColumnDefaultKind::Alias, alias};
columns.aliases.emplace_back(desc.alias, type);
}
other_columns.emplace_back(desc.column_name, type);
sample_block.insert(ColumnWithTypeAndName(type, desc.column_name));
}
columns.ordinary.insert(columns.ordinary.end(), num_columns.begin(), num_columns.end());
columns.ordinary.insert(columns.ordinary.end(), cat_columns.begin(), cat_columns.end());
/// Order is important: first numeric columns, then categorial, then all others.
for (const auto & column : num_columns)
columns.add(DB::ColumnDescription(column.name, column.type));
for (const auto & column : cat_columns)
columns.add(DB::ColumnDescription(column.name, column.type));
for (const auto & column : other_columns)
{
DB::ColumnDescription column_desc(column.name, column.type);
/// We assign Materialized kind to the column so that it doesn't show in SELECT *.
/// Because the table is readonly, we do not need default expression.
column_desc.default_desc.kind = ColumnDefaultKind::Materialized;
columns.add(std::move(column_desc));
}
for (auto & desc : columns_description)
{
if (!desc.alias.empty())
{
DB::ColumnDescription column(desc.alias, get_type(desc.column_type));
column.default_desc.kind = ColumnDefaultKind::Alias;
column.default_desc.expression = std::make_shared<ASTIdentifier>(desc.column_name);
columns.add(std::move(column));
}
}
setColumns(columns);
}

2
dbms/src/Storages/StorageDictionary.cpp
View File

@ -78,7 +78,7 @@ void StorageDictionary::checkNamesAndTypesCompatibleWithDictionary(const Diction
auto dictionary_names_and_types = getNamesAndTypes(dictionary_structure);
std::set<NameAndTypePair> names_and_types_set(dictionary_names_and_types.begin(), dictionary_names_and_types.end());
for (const auto & column : getColumns().ordinary)
for (const auto & column : getColumns().getOrdinary())
{
if (names_and_types_set.find(column) == names_and_types_set.end())
{

5
dbms/src/Storages/StorageFile.cpp
View File

@ -192,9 +192,10 @@ BlockInputStreams StorageFile::read(
{
BlockInputStreamPtr block_input = std::make_shared<StorageFileBlockInputStream>(*this, context, max_block_size);
const ColumnsDescription & columns = getColumns();
if (columns.defaults.empty())
auto column_defaults = columns.getDefaults();
if (column_defaults.empty())
return {block_input};
return {std::make_shared<AddingDefaultsBlockInputStream>(block_input, columns.defaults, context)};
return {std::make_shared<AddingDefaultsBlockInputStream>(block_input, column_defaults, context)};
}

4
dbms/src/Storages/StorageLog.cpp
View File

@ -551,8 +551,8 @@ void StorageLog::truncate(const ASTPtr &, const Context &)
const StorageLog::Marks & StorageLog::getMarksWithRealRowCount() const
{
const String & column_name = getColumns().ordinary.front().name;
const IDataType & column_type = *getColumns().ordinary.front().type;
const String & column_name = getColumns().begin()->name;
const IDataType & column_type = *getColumns().begin()->type;
String filename;
/** We take marks from first column.

2
dbms/src/Storages/StorageMySQL.cpp
View File

@ -57,7 +57,7 @@ BlockInputStreams StorageMySQL::read(
{
check(column_names);
String query = transformQueryForExternalDatabase(
*query_info.query, getColumns().ordinary, IdentifierQuotingStyle::Backticks, remote_database_name, remote_table_name, context);
*query_info.query, getColumns().getOrdinary(), IdentifierQuotingStyle::Backticks, remote_database_name, remote_table_name, context);
Block sample_block;
for (const String & column_name : column_names)

4
dbms/src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -397,8 +397,8 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo
{
if (allow_alter &&
(skip_sanity_checks ||
old_columns.ordinary.sizeOfDifference(columns_from_zk.ordinary) +
old_columns.materialized.sizeOfDifference(columns_from_zk.materialized) <= 2))
old_columns.getOrdinary().sizeOfDifference(columns_from_zk.getOrdinary()) +
old_columns.getMaterialized().sizeOfDifference(columns_from_zk.getMaterialized()) <= 2))
{
LOG_WARNING(log, "Table structure in ZooKeeper is a little different from local table structure. Assuming ALTER.");

6
dbms/src/Storages/StorageURL.cpp
View File

@ -176,10 +176,10 @@ BlockInputStreams IStorageURLBase::read(const Names & column_names,
ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef()));
const ColumnsDescription & columns = getColumns();
if (columns.defaults.empty())
auto column_defaults = getColumns().getDefaults();
if (column_defaults.empty())
return {block_input};
return {std::make_shared<AddingDefaultsBlockInputStream>(block_input, columns.defaults, context)};
return {std::make_shared<AddingDefaultsBlockInputStream>(block_input, column_defaults, context)};
}
void IStorageURLBase::rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/) {}

2
dbms/src/Storages/StorageXDBC.cpp
View File

@ -64,7 +64,7 @@ std::function<void(std::ostream &)> StorageXDBC::getReadPOSTDataCallback(const N
size_t /*max_block_size*/) const
{
String query = transformQueryForExternalDatabase(*query_info.query,
getColumns().ordinary,
getColumns().getOrdinary(),
bridge_helper->getIdentifierQuotingStyle(),
remote_database_name,
remote_table_name,

66
dbms/src/Storages/System/StorageSystemColumns.cpp
View File

@ -88,10 +88,7 @@ protected:
const std::string table_name = (*tables)[db_table_num].get<std::string>();
++db_table_num;
NamesAndTypesList columns;
ColumnDefaults column_defaults;
ColumnComments column_comments;
ColumnCodecs column_codecs;
ColumnsDescription columns;
Names cols_required_for_partition_key;
Names cols_required_for_sorting_key;
Names cols_required_for_primary_key;
@ -119,10 +116,7 @@ protected:
throw;
}
columns = storage->getColumns().getAll();
column_codecs = storage->getColumns().codecs;
column_defaults = storage->getColumns().defaults;
column_comments = storage->getColumns().comments;
columns = storage->getColumns();
cols_required_for_partition_key = storage->getColumnsRequiredForPartitionKey();
cols_required_for_sorting_key = storage->getColumnsRequiredForSortingKey();
@ -156,22 +150,19 @@ protected:
if (columns_mask[src_index++])
res_columns[res_index++]->insert(column.type->getName());
if (column.default_desc.expression)
{
const auto it = column_defaults.find(column.name);
if (it == std::end(column_defaults))
{
if (columns_mask[src_index++])
res_columns[res_index++]->insertDefault();
if (columns_mask[src_index++])
res_columns[res_index++]->insertDefault();
}
else
{
if (columns_mask[src_index++])
res_columns[res_index++]->insert(toString(it->second.kind));
if (columns_mask[src_index++])
res_columns[res_index++]->insert(queryToString(it->second.expression));
}
if (columns_mask[src_index++])
res_columns[res_index++]->insert(toString(column.default_desc.kind));
if (columns_mask[src_index++])
res_columns[res_index++]->insert(queryToString(column.default_desc.expression));
}
else
{
if (columns_mask[src_index++])
res_columns[res_index++]->insertDefault();
if (columns_mask[src_index++])
res_columns[res_index++]->insertDefault();
}
{
@ -196,19 +187,8 @@ protected:
}
}
{
const auto it = column_comments.find(column.name);
if (it == std::end(column_comments))
{
if (columns_mask[src_index++])
res_columns[res_index++]->insertDefault();
}
else
{
if (columns_mask[src_index++])
res_columns[res_index++]->insert(it->second);
}
}
if (columns_mask[src_index++])
res_columns[res_index++]->insert(column.comment);
{
auto find_in_vector = [&key = column.name](const Names& names)
@ -226,18 +206,12 @@ protected:
res_columns[res_index++]->insert(find_in_vector(cols_required_for_sampling));
}
if (columns_mask[src_index++])
{
const auto it = column_codecs.find(column.name);
if (it == std::end(column_codecs))
{
if (columns_mask[src_index++])
res_columns[res_index++]->insertDefault();
}
if (column.codec)
res_columns[res_index++]->insert("CODEC(" + column.codec->getCodecDesc() + ")");
else
{
if (columns_mask[src_index++])
res_columns[res_index++]->insert("CODEC(" + it->second->getCodecDesc() + ")");
}
res_columns[res_index++]->insertDefault();
}
++rows_count;

24
dbms/src/Storages/System/StorageSystemPartsBase.cpp
View File

@ -286,31 +286,21 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const
StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_)
: name(std::move(name_))
{
NamesAndTypesList aliases;
ColumnDefaults defaults;
ColumnsDescription columns(std::move(columns_));
auto add_alias = [&](const String & alias_name, const String & column_name)
{
DataTypePtr type;
for (const NameAndTypePair & col : columns_)
{
if (col.name == column_name)
{
type = col.type;
break;
}
}
if (!type)
throw Exception("No column " + column_name + " in table system." + name, ErrorCodes::LOGICAL_ERROR);
aliases.push_back({alias_name, type});
defaults[alias_name] = ColumnDefault{ColumnDefaultKind::Alias, std::make_shared<ASTIdentifier>(column_name)};
ColumnDescription column(alias_name, columns.get(column_name).type);
column.default_desc.kind = ColumnDefaultKind::Alias;
column.default_desc.expression = std::make_shared<ASTIdentifier>(column_name);
columns.add(column);
};
/// Add aliases for old column names for backwards compatibility.
add_alias("bytes", "bytes_on_disk");
add_alias("marks_size", "marks_bytes");
setColumns(ColumnsDescription(std::move(columns_), {}, std::move(aliases), std::move(defaults), ColumnComments{}, ColumnCodecs{}));
setColumns(columns);
}
}

13
dbms/src/Storages/System/StorageSystemPartsColumns.cpp
View File

@ -67,19 +67,14 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
String default_expression;
};
NamesAndTypesList columns_list = info.storage->getColumns().getAll();
const auto & column_defaults = info.storage->getColumns().defaults;
std::unordered_map<String, ColumnInfo> columns_info;
for (const auto & column : columns_list)
for (const auto & column : info.storage->getColumns())
{
ColumnInfo column_info;
const auto it = column_defaults.find(column.name);
if (it != std::end(column_defaults))
if (column.default_desc.expression)
{
column_info.default_kind = toString(it->second.kind);
column_info.default_expression = queryToString(it->second.expression);
column_info.default_kind = toString(column.default_desc.kind);
column_info.default_expression = queryToString(column.default_desc.expression);
}
columns_info[column.name] = column_info;

30
dbms/src/Storages/getStructureOfRemoteTable.cpp
View File

@ -89,29 +89,23 @@ ColumnsDescription getStructureOfRemoteTable(
for (size_t i = 0; i < size; ++i)
{
String column_name = (*name)[i].get<const String &>();
ColumnDescription column;
column.name = (*name)[i].get<const String &>();
String data_type_name = (*type)[i].get<const String &>();
column.type = data_type_factory.get(data_type_name);
String kind_name = (*default_kind)[i].get<const String &>();
auto data_type = data_type_factory.get(data_type_name);
if (kind_name.empty())
res.ordinary.emplace_back(column_name, std::move(data_type));
else
if (!kind_name.empty())
{
auto kind = columnDefaultKindFromString(kind_name);
column.default_desc.kind = columnDefaultKindFromString(kind_name);
String expr_str = (*default_expr)[i].get<const String &>();
ASTPtr expr = parseQuery(expr_parser, expr_str.data(), expr_str.data() + expr_str.size(), "default expression", 0);
res.defaults.emplace(column_name, ColumnDefault{kind, expr});
if (ColumnDefaultKind::Default == kind)
res.ordinary.emplace_back(column_name, std::move(data_type));
else if (ColumnDefaultKind::Materialized == kind)
res.materialized.emplace_back(column_name, std::move(data_type));
else if (ColumnDefaultKind::Alias == kind)
res.aliases.emplace_back(column_name, std::move(data_type));
column.default_desc.expression = parseQuery(
expr_parser, expr_str.data(), expr_str.data() + expr_str.size(), "default expression", 0);
}
res.add(column);
}
}