2017-04-01 09:19:00 +00:00
# include <Storages/AlterCommands.h>
# include <Storages/IStorage.h>
2018-06-13 13:49:27 +00:00
# include <DataTypes/DataTypeFactory.h>
2017-04-01 09:19:00 +00:00
# include <DataTypes/DataTypesNumber.h>
2017-12-25 18:58:39 +00:00
# include <DataTypes/NestedUtils.h>
2017-04-01 09:19:00 +00:00
# include <Interpreters/Context.h>
# include <Interpreters/ExpressionAnalyzer.h>
# include <Interpreters/ExpressionActions.h>
# include <Parsers/ASTIdentifier.h>
# include <Parsers/ASTExpressionList.h>
# include <Parsers/ASTLiteral.h>
# include <Parsers/ASTFunction.h>
2018-06-13 13:49:27 +00:00
# include <Parsers/ASTAlterQuery.h>
# include <Parsers/ASTColumnDeclaration.h>
# include <Common/typeid_cast.h>
2014-10-16 13:37:01 +00:00
2015-10-20 16:22:08 +00:00
2014-10-16 13:37:01 +00:00
namespace DB
{
2016-01-11 21:46:36 +00:00
namespace ErrorCodes
{
2017-04-01 07:20:54 +00:00
extern const int ILLEGAL_COLUMN ;
extern const int LOGICAL_ERROR ;
2016-01-11 21:46:36 +00:00
}
2018-06-13 13:49:27 +00:00
std : : optional < AlterCommand > AlterCommand : : parse ( const ASTAlterCommand * command_ast )
{
const DataTypeFactory & data_type_factory = DataTypeFactory : : instance ( ) ;
if ( command_ast - > type = = ASTAlterCommand : : ADD_COLUMN )
{
AlterCommand command ;
command . type = AlterCommand : : ADD_COLUMN ;
const auto & ast_col_decl = typeid_cast < const ASTColumnDeclaration & > ( * command_ast - > col_decl ) ;
command . column_name = ast_col_decl . name ;
if ( ast_col_decl . type )
{
command . data_type = data_type_factory . get ( ast_col_decl . type ) ;
}
if ( ast_col_decl . default_expression )
{
command . default_kind = columnDefaultKindFromString ( ast_col_decl . default_specifier ) ;
command . default_expression = ast_col_decl . default_expression ;
}
if ( command_ast - > column )
command . after_column = typeid_cast < const ASTIdentifier & > ( * command_ast - > column ) . name ;
return command ;
}
else if ( command_ast - > type = = ASTAlterCommand : : DROP_COLUMN & & ! command_ast - > partition )
{
if ( command_ast - > clear_column )
throw Exception ( " \" ALTER TABLE table CLEAR COLUMN column \" queries are not supported yet. Use \" CLEAR COLUMN column IN PARTITION \" . " , ErrorCodes : : NOT_IMPLEMENTED ) ;
AlterCommand command ;
command . type = AlterCommand : : DROP_COLUMN ;
command . column_name = typeid_cast < const ASTIdentifier & > ( * ( command_ast - > column ) ) . name ;
return command ;
}
else if ( command_ast - > type = = ASTAlterCommand : : MODIFY_COLUMN )
{
AlterCommand command ;
command . type = AlterCommand : : MODIFY_COLUMN ;
const auto & ast_col_decl = typeid_cast < const ASTColumnDeclaration & > ( * command_ast - > col_decl ) ;
command . column_name = ast_col_decl . name ;
if ( ast_col_decl . type )
{
command . data_type = data_type_factory . get ( ast_col_decl . type ) ;
}
if ( ast_col_decl . default_expression )
{
command . default_kind = columnDefaultKindFromString ( ast_col_decl . default_specifier ) ;
command . default_expression = ast_col_decl . default_expression ;
}
return command ;
}
else if ( command_ast - > type = = ASTAlterCommand : : MODIFY_PRIMARY_KEY )
{
AlterCommand command ;
command . type = AlterCommand : : MODIFY_PRIMARY_KEY ;
command . primary_key = command_ast - > primary_key ;
return command ;
}
2018-10-15 18:47:47 +00:00
else if ( command_ast - > type = = ASTAlterCommand : : MODIFY_ORDER_BY )
{
AlterCommand command ;
command . type = AlterCommand : : MODIFY_ORDER_BY ;
command . sorting_key = command_ast - > sorting_key ;
return command ;
}
2018-06-13 13:49:27 +00:00
else
return { } ;
}
/// the names are the same if they match the whole name or name_without_dot matches the part of the name up to the dot
static bool namesEqual ( const String & name_without_dot , const DB : : NameAndTypePair & name_type )
{
String name_with_dot = name_without_dot + " . " ;
return ( name_with_dot = = name_type . name . substr ( 0 , name_without_dot . length ( ) + 1 ) | | name_without_dot = = name_type . name ) ;
}
2018-03-06 20:18:34 +00:00
void AlterCommand : : apply ( ColumnsDescription & columns_description ) const
2016-01-11 21:46:36 +00:00
{
2017-04-01 07:20:54 +00:00
if ( type = = ADD_COLUMN )
{
2018-03-10 17:03:57 +00:00
if ( columns_description . getAll ( ) . contains ( column_name ) )
2018-05-07 02:01:11 +00:00
throw Exception { " Cannot add column " + column_name + " : column with this name already exists " , ErrorCodes : : ILLEGAL_COLUMN } ;
2017-04-01 07:20:54 +00:00
2017-12-25 21:57:29 +00:00
const auto add_column = [ this ] ( NamesAndTypesList & columns )
2017-04-01 07:20:54 +00:00
{
auto insert_it = columns . end ( ) ;
if ( ! after_column . empty ( ) )
{
/// We are trying to find first column from end with name `column_name` or with a name beginning with `column_name` and ".".
/// For example "fruits.bananas"
/// names are considered the same if they completely match or `name_without_dot` matches the part of the name to the point
const auto reverse_insert_it = std : : find_if ( columns . rbegin ( ) , columns . rend ( ) ,
std : : bind ( namesEqual , std : : cref ( after_column ) , std : : placeholders : : _1 ) ) ;
if ( reverse_insert_it = = columns . rend ( ) )
throw Exception ( " Wrong column name. Cannot find column " + after_column + " to insert after " ,
2018-02-22 00:52:30 +00:00
ErrorCodes : : ILLEGAL_COLUMN ) ;
2017-04-01 07:20:54 +00:00
else
{
/// base returns an iterator that is already offset by one element to the right
insert_it = reverse_insert_it . base ( ) ;
}
}
columns . emplace ( insert_it , column_name , data_type ) ;
} ;
2018-03-12 13:47:01 +00:00
if ( default_kind = = ColumnDefaultKind : : Default )
2018-03-06 20:18:34 +00:00
add_column ( columns_description . ordinary ) ;
2018-03-12 13:47:01 +00:00
else if ( default_kind = = ColumnDefaultKind : : Materialized )
2018-03-06 20:18:34 +00:00
add_column ( columns_description . materialized ) ;
2018-03-12 13:47:01 +00:00
else if ( default_kind = = ColumnDefaultKind : : Alias )
2018-03-06 20:18:34 +00:00
add_column ( columns_description . aliases ) ;
2017-04-01 07:20:54 +00:00
else
2018-03-12 13:47:01 +00:00
throw Exception { " Unknown ColumnDefaultKind value " , ErrorCodes : : LOGICAL_ERROR } ;
2017-04-01 07:20:54 +00:00
if ( default_expression )
2018-03-12 13:47:01 +00:00
columns_description . defaults . emplace ( column_name , ColumnDefault { default_kind , default_expression } ) ;
2017-04-01 07:20:54 +00:00
/// Slow, because each time a list is copied
2018-03-06 20:18:34 +00:00
columns_description . ordinary = Nested : : flatten ( columns_description . ordinary ) ;
2017-04-01 07:20:54 +00:00
}
else if ( type = = DROP_COLUMN )
{
/// look for a column in list and remove it if present, also removing corresponding entry from column_defaults
2018-03-06 20:18:34 +00:00
const auto remove_column = [ & columns_description , this ] ( NamesAndTypesList & columns )
2018-01-10 00:04:08 +00:00
{
2017-04-01 07:20:54 +00:00
auto removed = false ;
2017-12-25 21:57:29 +00:00
NamesAndTypesList : : iterator column_it ;
2017-04-01 07:20:54 +00:00
while ( columns . end ( ) ! = ( column_it = std : : find_if ( columns . begin ( ) , columns . end ( ) ,
std : : bind ( namesEqual , std : : cref ( column_name ) , std : : placeholders : : _1 ) ) ) )
{
removed = true ;
column_it = columns . erase ( column_it ) ;
2018-03-06 20:18:34 +00:00
columns_description . defaults . erase ( column_name ) ;
2017-04-01 07:20:54 +00:00
}
return removed ;
} ;
2018-03-06 20:18:34 +00:00
if ( ! remove_column ( columns_description . ordinary ) & &
! remove_column ( columns_description . materialized ) & &
! remove_column ( columns_description . aliases ) )
2017-04-01 07:20:54 +00:00
{
throw Exception ( " Wrong column name. Cannot find column " + column_name + " to drop " ,
2018-02-22 00:52:30 +00:00
ErrorCodes : : ILLEGAL_COLUMN ) ;
2017-04-01 07:20:54 +00:00
}
}
else if ( type = = MODIFY_COLUMN )
{
2018-03-06 20:18:34 +00:00
const auto default_it = columns_description . defaults . find ( column_name ) ;
const auto had_default_expr = default_it ! = std : : end ( columns_description . defaults ) ;
2018-03-12 13:47:01 +00:00
const auto old_default_kind = had_default_expr ? default_it - > second . kind : ColumnDefaultKind { } ;
2017-04-01 07:20:54 +00:00
/// target column list
2018-03-06 20:18:34 +00:00
auto & new_columns =
2018-03-12 13:47:01 +00:00
default_kind = = ColumnDefaultKind : : Default ? columns_description . ordinary
: default_kind = = ColumnDefaultKind : : Materialized ? columns_description . materialized
2018-03-06 20:18:34 +00:00
: columns_description . aliases ;
2017-04-01 07:20:54 +00:00
/// find column or throw exception
2018-01-10 00:04:08 +00:00
const auto find_column = [ this ] ( NamesAndTypesList & columns )
{
2017-04-01 07:20:54 +00:00
const auto it = std : : find_if ( columns . begin ( ) , columns . end ( ) ,
std : : bind ( namesEqual , std : : cref ( column_name ) , std : : placeholders : : _1 ) ) ;
if ( it = = columns . end ( ) )
throw Exception ( " Wrong column name. Cannot find column " + column_name + " to modify " ,
2018-02-22 00:52:30 +00:00
ErrorCodes : : ILLEGAL_COLUMN ) ;
2017-04-01 07:20:54 +00:00
return it ;
} ;
/// if default types differ, remove column from the old list, then add to the new list
2018-03-12 13:47:01 +00:00
if ( default_kind ! = old_default_kind )
2017-04-01 07:20:54 +00:00
{
/// source column list
2018-03-06 20:18:34 +00:00
auto & old_columns =
2018-03-12 13:47:01 +00:00
old_default_kind = = ColumnDefaultKind : : Default ? columns_description . ordinary
: old_default_kind = = ColumnDefaultKind : : Materialized ? columns_description . materialized
2018-03-06 20:18:34 +00:00
: columns_description . aliases ;
2017-04-01 07:20:54 +00:00
const auto old_column_it = find_column ( old_columns ) ;
new_columns . emplace_back ( * old_column_it ) ;
old_columns . erase ( old_column_it ) ;
/// do not forget to change the default type of old column
if ( had_default_expr )
2018-03-12 13:47:01 +00:00
columns_description . defaults [ column_name ] . kind = default_kind ;
2017-04-01 07:20:54 +00:00
}
/// find column in one of three column lists
const auto column_it = find_column ( new_columns ) ;
column_it - > type = data_type ;
if ( ! default_expression & & had_default_expr )
/// new column has no default expression, remove it from column_defaults along with it's type
2018-03-06 20:18:34 +00:00
columns_description . defaults . erase ( column_name ) ;
2017-04-01 07:20:54 +00:00
else if ( default_expression & & ! had_default_expr )
/// new column has a default expression while the old one had not, add it it column_defaults
2018-03-12 13:47:01 +00:00
columns_description . defaults . emplace ( column_name , ColumnDefault { default_kind , default_expression } ) ;
2017-04-01 07:20:54 +00:00
else if ( had_default_expr )
/// both old and new columns have default expression, update it
2018-03-06 20:18:34 +00:00
columns_description . defaults [ column_name ] . expression = default_expression ;
2017-04-01 07:20:54 +00:00
}
2018-10-15 18:47:47 +00:00
else if ( type = = MODIFY_PRIMARY_KEY | | type = = MODIFY_ORDER_BY )
2017-04-01 07:20:54 +00:00
{
/// This have no relation to changing the list of columns.
/// TODO Check that all columns exist, that only columns with constant defaults are added.
}
else
throw Exception ( " Wrong parameter type in ALTER query " , ErrorCodes : : LOGICAL_ERROR ) ;
2016-01-11 21:46:36 +00:00
}
2014-10-21 12:11:20 +00:00
2016-05-05 18:28:46 +00:00
2018-03-06 20:18:34 +00:00
void AlterCommands : : apply ( ColumnsDescription & columns_description ) const
2016-01-11 21:46:36 +00:00
{
2018-03-06 20:18:34 +00:00
auto new_columns_description = columns_description ;
2017-04-01 07:20:54 +00:00
for ( const AlterCommand & command : * this )
2018-03-06 20:18:34 +00:00
command . apply ( new_columns_description ) ;
2017-04-01 07:20:54 +00:00
2018-03-06 20:18:34 +00:00
columns_description = std : : move ( new_columns_description ) ;
2016-01-11 21:46:36 +00:00
}
2014-10-21 12:11:20 +00:00
2018-05-15 12:56:14 +00:00
void AlterCommands : : validate ( const IStorage & table , const Context & context )
2016-01-11 21:46:36 +00:00
{
2018-05-15 12:56:14 +00:00
auto all_columns = table . getColumns ( ) . getAll ( ) ;
auto defaults = table . getColumns ( ) . defaults ;
2017-04-01 07:20:54 +00:00
2017-12-25 21:57:29 +00:00
std : : vector < std : : pair < NameAndTypePair , AlterCommand * > > defaulted_columns { } ;
2017-04-01 07:20:54 +00:00
auto default_expr_list = std : : make_shared < ASTExpressionList > ( ) ;
default_expr_list - > children . reserve ( defaults . size ( ) ) ;
for ( AlterCommand & command : * this )
{
if ( command . type = = AlterCommand : : ADD_COLUMN | | command . type = = AlterCommand : : MODIFY_COLUMN )
{
const auto & column_name = command . column_name ;
2018-03-06 20:18:34 +00:00
const auto column_it = std : : find_if ( std : : begin ( all_columns ) , std : : end ( all_columns ) ,
2018-06-13 13:49:27 +00:00
std : : bind ( namesEqual , std : : cref ( command . column_name ) , std : : placeholders : : _1 ) ) ;
2017-04-01 07:20:54 +00:00
if ( command . type = = AlterCommand : : ADD_COLUMN )
{
2018-03-06 20:18:34 +00:00
if ( std : : end ( all_columns ) ! = column_it )
2018-05-07 02:01:11 +00:00
throw Exception { " Cannot add column " + column_name + " : column with this name already exists " , ErrorCodes : : ILLEGAL_COLUMN } ;
2017-04-01 07:20:54 +00:00
}
else if ( command . type = = AlterCommand : : MODIFY_COLUMN )
{
2018-03-06 20:18:34 +00:00
if ( std : : end ( all_columns ) = = column_it )
2018-05-07 02:01:11 +00:00
throw Exception { " Wrong column name. Cannot find column " + column_name + " to modify " , ErrorCodes : : ILLEGAL_COLUMN } ;
2017-04-01 07:20:54 +00:00
2018-03-06 20:18:34 +00:00
all_columns . erase ( column_it ) ;
2017-04-01 07:20:54 +00:00
defaults . erase ( column_name ) ;
}
/// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions
2018-03-06 20:18:34 +00:00
all_columns . emplace_back ( column_name , command . data_type ? command . data_type : std : : make_shared < DataTypeUInt8 > ( ) ) ;
2017-04-01 07:20:54 +00:00
if ( command . default_expression )
{
if ( command . data_type )
{
const auto & final_column_name = column_name ;
const auto tmp_column_name = final_column_name + " _tmp " ;
const auto column_type_raw_ptr = command . data_type . get ( ) ;
default_expr_list - > children . emplace_back ( setAlias (
2018-02-26 03:37:08 +00:00
makeASTFunction ( " CAST " , std : : make_shared < ASTIdentifier > ( tmp_column_name ) ,
std : : make_shared < ASTLiteral > ( Field ( column_type_raw_ptr - > getName ( ) ) ) ) ,
2017-04-01 07:20:54 +00:00
final_column_name ) ) ;
default_expr_list - > children . emplace_back ( setAlias ( command . default_expression - > clone ( ) , tmp_column_name ) ) ;
2017-12-25 21:57:29 +00:00
defaulted_columns . emplace_back ( NameAndTypePair { column_name , command . data_type } , & command ) ;
2017-04-01 07:20:54 +00:00
}
else
{
/// no type explicitly specified, will deduce later
default_expr_list - > children . emplace_back (
setAlias ( command . default_expression - > clone ( ) , column_name ) ) ;
2017-12-25 21:57:29 +00:00
defaulted_columns . emplace_back ( NameAndTypePair { column_name , nullptr } , & command ) ;
2017-04-01 07:20:54 +00:00
}
}
}
else if ( command . type = = AlterCommand : : DROP_COLUMN )
{
2018-02-22 00:52:30 +00:00
for ( const auto & default_column : defaults )
2018-02-21 00:49:16 +00:00
{
2018-02-22 00:52:30 +00:00
const auto & default_expression = default_column . second . expression ;
2018-03-06 20:18:34 +00:00
const auto actions = ExpressionAnalyzer { default_expression , context , { } , all_columns } . getActions ( true ) ;
2018-03-02 19:32:30 +00:00
const auto required_columns = actions - > getRequiredColumns ( ) ;
2018-02-21 00:49:16 +00:00
2018-03-02 19:32:30 +00:00
if ( required_columns . end ( ) ! = std : : find ( required_columns . begin ( ) , required_columns . end ( ) , command . column_name ) )
2018-02-21 00:49:16 +00:00
throw Exception (
2018-02-22 00:52:30 +00:00
" Cannot drop column " + command . column_name + " , because column " + default_column . first +
2018-03-02 19:32:30 +00:00
" depends on it " , ErrorCodes : : ILLEGAL_COLUMN ) ;
2018-02-21 00:49:16 +00:00
}
2018-03-02 19:32:30 +00:00
2017-04-01 07:20:54 +00:00
auto found = false ;
2018-03-06 20:18:34 +00:00
for ( auto it = std : : begin ( all_columns ) ; it ! = std : : end ( all_columns ) ; )
2018-03-02 19:32:30 +00:00
{
2018-06-13 13:49:27 +00:00
if ( namesEqual ( command . column_name , * it ) )
2017-04-01 07:20:54 +00:00
{
found = true ;
2018-03-06 20:18:34 +00:00
it = all_columns . erase ( it ) ;
2017-04-01 07:20:54 +00:00
}
else
+ + it ;
2018-03-02 19:32:30 +00:00
}
2017-04-01 07:20:54 +00:00
for ( auto it = std : : begin ( defaults ) ; it ! = std : : end ( defaults ) ; )
2018-03-02 19:32:30 +00:00
{
2018-06-13 13:49:27 +00:00
if ( namesEqual ( command . column_name , { it - > first , nullptr } ) )
2017-04-01 07:20:54 +00:00
it = defaults . erase ( it ) ;
else
+ + it ;
2018-03-02 19:32:30 +00:00
}
2017-04-01 07:20:54 +00:00
if ( ! found )
throw Exception ( " Wrong column name. Cannot find column " + command . column_name + " to drop " ,
2018-02-22 00:52:30 +00:00
ErrorCodes : : ILLEGAL_COLUMN ) ;
2017-04-01 07:20:54 +00:00
}
}
/** Existing defaulted columns may require default expression extensions with a type conversion,
* therefore we add them to defaulted_columns to allow further processing */
for ( const auto & col_def : defaults )
{
const auto & column_name = col_def . first ;
2018-03-06 20:18:34 +00:00
const auto column_it = std : : find_if ( all_columns . begin ( ) , all_columns . end ( ) , [ & ] ( const NameAndTypePair & name_type )
2018-06-13 13:49:27 +00:00
{ return namesEqual ( column_name , name_type ) ; } ) ;
2017-04-01 07:20:54 +00:00
const auto tmp_column_name = column_name + " _tmp " ;
const auto & column_type_ptr = column_it - > type ;
default_expr_list - > children . emplace_back ( setAlias (
2018-02-26 03:37:08 +00:00
makeASTFunction ( " CAST " , std : : make_shared < ASTIdentifier > ( tmp_column_name ) ,
std : : make_shared < ASTLiteral > ( Field ( column_type_ptr - > getName ( ) ) ) ) ,
2017-04-01 07:20:54 +00:00
column_name ) ) ;
default_expr_list - > children . emplace_back ( setAlias ( col_def . second . expression - > clone ( ) , tmp_column_name ) ) ;
2017-12-25 21:57:29 +00:00
defaulted_columns . emplace_back ( NameAndTypePair { column_name , column_type_ptr } , nullptr ) ;
2017-04-01 07:20:54 +00:00
}
2018-03-06 20:18:34 +00:00
const auto actions = ExpressionAnalyzer { default_expr_list , context , { } , all_columns } . getActions ( true ) ;
2017-04-01 07:20:54 +00:00
const auto block = actions - > getSampleBlock ( ) ;
/// set deduced types, modify default expression if necessary
for ( auto & defaulted_column : defaulted_columns )
{
const auto & name_and_type = defaulted_column . first ;
AlterCommand * & command_ptr = defaulted_column . second ;
const auto & column_name = name_and_type . name ;
const auto has_explicit_type = nullptr ! = name_and_type . type ;
/// default expression on old column
if ( has_explicit_type )
{
const auto & explicit_type = name_and_type . type ;
const auto & tmp_column = block . getByName ( column_name + " _tmp " ) ;
const auto & deduced_type = tmp_column . type ;
// column not specified explicitly in the ALTER query may require default_expression modification
2017-12-23 01:55:46 +00:00
if ( ! explicit_type - > equals ( * deduced_type ) )
2017-04-01 07:20:54 +00:00
{
const auto default_it = defaults . find ( column_name ) ;
/// column has no associated alter command, let's create it
if ( ! command_ptr )
{
/// add a new alter command to modify existing column
this - > emplace_back ( AlterCommand {
AlterCommand : : MODIFY_COLUMN , column_name , explicit_type ,
2018-03-12 13:47:01 +00:00
default_it - > second . kind , default_it - > second . expression
2017-04-01 07:20:54 +00:00
} ) ;
command_ptr = & this - > back ( ) ;
}
command_ptr - > default_expression = makeASTFunction ( " CAST " , command_ptr - > default_expression - > clone ( ) ,
2018-02-26 03:37:08 +00:00
std : : make_shared < ASTLiteral > ( Field ( explicit_type - > getName ( ) ) ) ) ;
2017-04-01 07:20:54 +00:00
}
}
else
{
/// just set deduced type
command_ptr - > data_type = block . getByName ( column_name ) . type ;
}
}
2014-10-16 13:37:01 +00:00
}
2016-01-11 21:46:36 +00:00
}