thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11516 from ClickHouse/consistent_metadata4

Browse Source 
Storage metadata as a single struct.
This commit is contained in:
alesapin 2020-06-16 13:28:22 +03:00 committed by GitHub
commit 362aaf1385
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
44 changed files with 1069 additions and 720 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
programs/copier/Internals.cpp
View File

@ -1,5 +1,6 @@
#include "Internals.h"
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/extractKeyExpressionList.h>
namespace DB
{
@ -184,9 +185,9 @@ Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast)
const auto sorting_key_ast = extractOrderBy(storage_ast);
const auto primary_key_ast = extractPrimaryKey(storage_ast);
const auto sorting_key_expr_list = MergeTreeData::extractKeyExpressionList(sorting_key_ast);
const auto sorting_key_expr_list = extractKeyExpressionList(sorting_key_ast);
const auto primary_key_expr_list = primary_key_ast
? MergeTreeData::extractKeyExpressionList(primary_key_ast) : sorting_key_expr_list->clone();
? extractKeyExpressionList(primary_key_ast) : sorting_key_expr_list->clone();
/// Maybe we have to handle VersionedCollapsing engine separately. But in our case in looks pointless.

2
src/DataStreams/PushingToViewsBlockOutputStream.cpp
View File

@ -72,7 +72,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
StoragePtr inner_table = materialized_view->getTargetTable();
auto inner_table_id = inner_table->getStorageID();
query = materialized_view->getInnerQuery();
query = materialized_view->getSelectQuery().inner_query;
std::unique_ptr<ASTInsertQuery> insert = std::make_unique<ASTInsertQuery>();
insert->table_id = inner_table_id;

36
src/DataStreams/TTLBlockInputStream.cpp
View File

@ -70,21 +70,22 @@ TTLBlockInputStream::TTLBlockInputStream(
defaults_expression = ExpressionAnalyzer{default_expr_list, syntax_result, storage.global_context}.getActions(true);
}
if (storage.hasRowsTTL() && storage.getRowsTTL().mode == TTLMode::GROUP_BY)
auto storage_rows_ttl = storage.getRowsTTL();
if (storage.hasRowsTTL() && storage_rows_ttl.mode == TTLMode::GROUP_BY)
{
current_key_value.resize(storage.getRowsTTL().group_by_keys.size());
current_key_value.resize(storage_rows_ttl.group_by_keys.size());
ColumnNumbers keys;
for (const auto & key : storage.getRowsTTL().group_by_keys)
for (const auto & key : storage_rows_ttl.group_by_keys)
keys.push_back(header.getPositionByName(key));
agg_key_columns.resize(storage.getRowsTTL().group_by_keys.size());
agg_key_columns.resize(storage_rows_ttl.group_by_keys.size());
AggregateDescriptions aggregates = storage.getRowsTTL().aggregate_descriptions;
AggregateDescriptions aggregates = storage_rows_ttl.aggregate_descriptions;
for (auto & descr : aggregates)
if (descr.arguments.empty())
for (const auto & name : descr.argument_names)
descr.arguments.push_back(header.getPositionByName(name));
agg_aggregate_columns.resize(storage.getRowsTTL().aggregate_descriptions.size());
agg_aggregate_columns.resize(storage_rows_ttl.aggregate_descriptions.size());
const Settings & settings = storage.global_context.getSettingsRef();
@ -105,8 +106,9 @@ bool TTLBlockInputStream::isTTLExpired(time_t ttl) const
Block TTLBlockInputStream::readImpl()
{
/// Skip all data if table ttl is expired for part
if (storage.hasRowsTTL() && !storage.getRowsTTL().where_expression &&
storage.getRowsTTL().mode != TTLMode::GROUP_BY && isTTLExpired(old_ttl_infos.table_ttl.max))
auto storage_rows_ttl = storage.getRowsTTL();
if (storage.hasRowsTTL() && !storage_rows_ttl.where_expression &&
storage_rows_ttl.mode != TTLMode::GROUP_BY && isTTLExpired(old_ttl_infos.table_ttl.max))
{
rows_removed = data_part->rows_count;
return {};
@ -151,7 +153,7 @@ void TTLBlockInputStream::readSuffixImpl()
void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
{
const auto & rows_ttl = storage.getRowsTTL();
auto rows_ttl = storage.getRowsTTL();
rows_ttl.expression->execute(block);
if (rows_ttl.where_expression)
@ -160,8 +162,8 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
const IColumn * ttl_column =
block.getByName(rows_ttl.result_column).column.get();
const IColumn * where_result_column = storage.getRowsTTL().where_expression ?
block.getByName(storage.getRowsTTL().where_result_column).column.get() : nullptr;
const IColumn * where_result_column = rows_ttl.where_expression ?
block.getByName(rows_ttl.where_result_column).column.get() : nullptr;
const auto & column_names = header.getNames();
@ -199,6 +201,7 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
size_t rows_aggregated = 0;
size_t current_key_start = 0;
size_t rows_with_current_key = 0;
auto storage_rows_ttl = storage.getRowsTTL();
for (size_t i = 0; i < block.rows(); ++i)
{
UInt32 cur_ttl = getTimestampByIndex(ttl_column, i);
@ -206,9 +209,9 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
bool ttl_expired = isTTLExpired(cur_ttl) && where_filter_passed;
bool same_as_current = true;
for (size_t j = 0; j < storage.getRowsTTL().group_by_keys.size(); ++j)
for (size_t j = 0; j < storage_rows_ttl.group_by_keys.size(); ++j)
{
const String & key_column = storage.getRowsTTL().group_by_keys[j];
const String & key_column = storage_rows_ttl.group_by_keys[j];
const IColumn * values_column = block.getByName(key_column).column.get();
if (!same_as_current || (*values_column)[i] != current_key_value[j])
{
@ -275,17 +278,18 @@ void TTLBlockInputStream::finalizeAggregates(MutableColumns & result_columns)
if (!agg_result.empty())
{
auto aggregated_res = aggregator->convertToBlocks(agg_result, true, 1);
auto storage_rows_ttl = storage.getRowsTTL();
for (auto & agg_block : aggregated_res)
{
for (const auto & it : storage.getRowsTTL().set_parts)
for (const auto & it : storage_rows_ttl.set_parts)
it.expression->execute(agg_block);
for (const auto & name : storage.getRowsTTL().group_by_keys)
for (const auto & name : storage_rows_ttl.group_by_keys)
{
const IColumn * values_column = agg_block.getByName(name).column.get();
auto & result_column = result_columns[header.getPositionByName(name)];
result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
}
for (const auto & it : storage.getRowsTTL().set_parts)
for (const auto & it : storage_rows_ttl.set_parts)
{
const IColumn * values_column = agg_block.getByName(it.expression_result_column_name).column.get();
auto & result_column = result_columns[header.getPositionByName(it.column_name)];

20
src/Databases/DatabaseOrdinary.cpp
View File

@ -253,9 +253,9 @@ void DatabaseOrdinary::alterTable(
ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->indices, new_indices);
ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->constraints, new_constraints);
if (metadata.select)
if (metadata.select.select_query)
{
ast->replace(ast_create_query.select, metadata.select);
ast->replace(ast_create_query.select, metadata.select.select_query);
}
/// MaterializedView is one type of CREATE query without storage.
@ -263,17 +263,17 @@ void DatabaseOrdinary::alterTable(
{
ASTStorage & storage_ast = *ast_create_query.storage;
/// ORDER BY may change, but cannot appear, it's required construction
if (metadata.order_by_ast && storage_ast.order_by)
storage_ast.set(storage_ast.order_by, metadata.order_by_ast);
if (metadata.sorting_key.definition_ast && storage_ast.order_by)
storage_ast.set(storage_ast.order_by, metadata.sorting_key.definition_ast);
if (metadata.primary_key_ast)
storage_ast.set(storage_ast.primary_key, metadata.primary_key_ast);
if (metadata.primary_key.definition_ast)
storage_ast.set(storage_ast.primary_key, metadata.primary_key.definition_ast);
if (metadata.ttl_for_table_ast)
storage_ast.set(storage_ast.ttl_table, metadata.ttl_for_table_ast);
if (metadata.table_ttl.definition_ast)
storage_ast.set(storage_ast.ttl_table, metadata.table_ttl.definition_ast);
if (metadata.settings_ast)
storage_ast.set(storage_ast.settings, metadata.settings_ast);
if (metadata.settings_changes)
storage_ast.set(storage_ast.settings, metadata.settings_changes);
}
statement = getObjectDefinitionFromCreateQuery(ast);

2
src/Interpreters/ExpressionActions.h
View File

@ -163,6 +163,8 @@ public:
~ExpressionActions();
ExpressionActions(const ExpressionActions & other) = default;
/// Add the input column.
/// The name of the column must not match the names of the intermediate columns that occur when evaluating the expression.
/// The expression must not have any PROJECT actions.

57
src/Storages/AlterCommands.cpp
View File

@ -313,21 +313,27 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con
column.default_desc.expression = default_expression;
}
});
}
else if (type == MODIFY_ORDER_BY)
{
if (!metadata.primary_key_ast && metadata.order_by_ast)
auto & sorting_key = metadata.sorting_key;
auto & primary_key = metadata.primary_key;
if (primary_key.definition_ast == nullptr && sorting_key.definition_ast != nullptr)
{
/// Primary and sorting key become independent after this ALTER so we have to
/// save the old ORDER BY expression as the new primary key.
metadata.primary_key_ast = metadata.order_by_ast->clone();
/// Primary and sorting key become independent after this ALTER so
/// we have to save the old ORDER BY expression as the new primary
/// key.
primary_key = KeyDescription::getKeyFromAST(sorting_key.definition_ast, metadata.columns, context);
}
metadata.order_by_ast = order_by;
/// Recalculate key with new order_by expression.
sorting_key.recalculateWithNewAST(order_by, metadata.columns, context);
}
else if (type == COMMENT_COLUMN)
{
metadata.columns.modify(column_name, [&](ColumnDescription & column) { column.comment = *comment; });
metadata.columns.modify(column_name,
[&](ColumnDescription & column) { column.comment = *comment; });
}
else if (type == ADD_INDEX)
{
@ -430,15 +436,15 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con
}
else if (type == MODIFY_TTL)
{
metadata.ttl_for_table_ast = ttl;
metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl, metadata.columns, context, metadata.primary_key);
}
else if (type == MODIFY_QUERY)
{
metadata.select = select;
metadata.select = SelectQueryDescription::getSelectQueryFromASTForMatView(select, context);
}
else if (type == MODIFY_SETTING)
{
auto & settings_from_storage = metadata.settings_ast->as<ASTSetQuery &>().changes;
auto & settings_from_storage = metadata.settings_changes->as<ASTSetQuery &>().changes;
for (const auto & change : settings_changes)
{
auto finder = [&change](const SettingChange & c) { return c.name == change.name; };
@ -465,8 +471,11 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con
rename_visitor.visit(column_to_modify.ttl);
});
}
if (metadata.ttl_for_table_ast)
rename_visitor.visit(metadata.ttl_for_table_ast);
if (metadata.table_ttl.definition_ast)
rename_visitor.visit(metadata.table_ttl.definition_ast);
metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(
metadata.table_ttl.definition_ast, metadata.columns, context, metadata.primary_key);
for (auto & constraint : metadata.constraints.constraints)
rename_visitor.visit(constraint);
@ -707,6 +716,30 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, const Context & co
if (!command.ignore)
command.apply(metadata_copy, context);
/// Changes in columns may lead to changes in keys expression.
metadata_copy.sorting_key.recalculateWithNewColumns(metadata_copy.columns, context);
if (metadata_copy.primary_key.definition_ast != nullptr)
{
metadata_copy.primary_key.recalculateWithNewColumns(metadata_copy.columns, context);
}
else
{
metadata_copy.primary_key = KeyDescription::getKeyFromAST(metadata_copy.sorting_key.definition_ast, metadata_copy.columns, context);
metadata_copy.primary_key.definition_ast = nullptr;
}
/// Changes in columns may lead to changes in TTL expressions.
auto column_ttl_asts = metadata_copy.columns.getColumnTTLs();
for (const auto & [name, ast] : column_ttl_asts)
{
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, metadata_copy.columns, context, metadata_copy.primary_key);
metadata_copy.column_ttls_by_name[name] = new_ttl_entry;
}
if (metadata_copy.table_ttl.definition_ast != nullptr)
metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(
metadata_copy.table_ttl.definition_ast, metadata_copy.columns, context, metadata_copy.primary_key);
metadata = std::move(metadata_copy);
}
@ -832,7 +865,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con
}
else if (command.type == AlterCommand::MODIFY_SETTING)
{
if (metadata.settings_ast == nullptr)
if (metadata.settings_changes == nullptr)
throw Exception{"Cannot alter settings, because table engine doesn't support settings changes", ErrorCodes::BAD_ARGUMENTS};
}
else if (command.type == AlterCommand::RENAME_COLUMN)

15
src/Storages/ConstraintsDescription.cpp
View File

@ -54,4 +54,19 @@ ConstraintsExpressions ConstraintsDescription::getExpressions(const DB::Context
return res;
}
ConstraintsDescription::ConstraintsDescription(const ConstraintsDescription & other)
{
constraints.reserve(other.constraints.size());
for (const auto & constraint : other.constraints)
constraints.emplace_back(constraint->clone());
}
ConstraintsDescription & ConstraintsDescription::operator=(const ConstraintsDescription & other)
{
constraints.resize(other.constraints.size());
for (size_t i = 0; i < constraints.size(); ++i)
constraints[i] = other.constraints[i]->clone();
return *this;
}
}

3
src/Storages/ConstraintsDescription.h
View File

@ -20,6 +20,9 @@ struct ConstraintsDescription
static ConstraintsDescription parse(const String & str);
ConstraintsExpressions getExpressions(const Context & context, const NamesAndTypesList & source_columns_) const;
ConstraintsDescription(const ConstraintsDescription & other);
ConstraintsDescription & operator=(const ConstraintsDescription & other);
};
}

160
src/Storages/IStorage.cpp
View File

@ -34,23 +34,22 @@ namespace ErrorCodes
const ColumnsDescription & IStorage::getColumns() const
{
return columns;
return metadata.columns;
}
const IndicesDescription & IStorage::getSecondaryIndices() const
{
return secondary_indices;
return metadata.secondary_indices;
}
bool IStorage::hasSecondaryIndices() const
{
return !secondary_indices.empty();
return !metadata.secondary_indices.empty();
}
const ConstraintsDescription & IStorage::getConstraints() const
{
return constraints;
return metadata.constraints;
}
Block IStorage::getSampleBlock() const
@ -292,17 +291,17 @@ void IStorage::setColumns(ColumnsDescription columns_)
{
if (columns_.getOrdinary().empty())
throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
columns = std::move(columns_);
metadata.columns = std::move(columns_);
}
void IStorage::setSecondaryIndices(IndicesDescription secondary_indices_)
{
secondary_indices = std::move(secondary_indices_);
metadata.secondary_indices = std::move(secondary_indices_);
}
void IStorage::setConstraints(ConstraintsDescription constraints_)
{
constraints = std::move(constraints_);
metadata.constraints = std::move(constraints_);
}
bool IStorage::isVirtualColumn(const String & column_name) const
@ -373,11 +372,6 @@ TableStructureWriteLockHolder IStorage::lockExclusively(const String & query_id,
return result;
}
StorageInMemoryMetadata IStorage::getInMemoryMetadata() const
{
return StorageInMemoryMetadata(getColumns(), getSecondaryIndices(), getConstraints());
}
void IStorage::alter(
const AlterCommands & params,
const Context & context,
@ -385,14 +379,14 @@ void IStorage::alter(
{
lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout);
auto table_id = getStorageID();
StorageInMemoryMetadata metadata = getInMemoryMetadata();
params.apply(metadata, context);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata);
setColumns(std::move(metadata.columns));
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
params.apply(new_metadata, context);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata);
setColumns(std::move(new_metadata.columns));
}
void IStorage::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */)
void IStorage::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const
{
for (const auto & command : commands)
{
@ -421,137 +415,139 @@ NamesAndTypesList IStorage::getVirtuals() const
return {};
}
const StorageMetadataKeyField & IStorage::getPartitionKey() const
const KeyDescription & IStorage::getPartitionKey() const
{
return partition_key;
return metadata.partition_key;
}
void IStorage::setPartitionKey(const StorageMetadataKeyField & partition_key_)
void IStorage::setPartitionKey(const KeyDescription & partition_key_)
{
partition_key = partition_key_;
metadata.partition_key = partition_key_;
}
bool IStorage::isPartitionKeyDefined() const
{
return partition_key.definition_ast != nullptr;
return metadata.partition_key.definition_ast != nullptr;
}
bool IStorage::hasPartitionKey() const
{
return !partition_key.column_names.empty();
return !metadata.partition_key.column_names.empty();
}
Names IStorage::getColumnsRequiredForPartitionKey() const
{
if (hasPartitionKey())
return partition_key.expression->getRequiredColumns();
return metadata.partition_key.expression->getRequiredColumns();
return {};
}
const StorageMetadataKeyField & IStorage::getSortingKey() const
const KeyDescription & IStorage::getSortingKey() const
{
return sorting_key;
return metadata.sorting_key;
}
void IStorage::setSortingKey(const StorageMetadataKeyField & sorting_key_)
void IStorage::setSortingKey(const KeyDescription & sorting_key_)
{
sorting_key = sorting_key_;
metadata.sorting_key = sorting_key_;
}
bool IStorage::isSortingKeyDefined() const
{
return sorting_key.definition_ast != nullptr;
return metadata.sorting_key.definition_ast != nullptr;
}
bool IStorage::hasSortingKey() const
{
return !sorting_key.column_names.empty();
return !metadata.sorting_key.column_names.empty();
}
Names IStorage::getColumnsRequiredForSortingKey() const
{
if (hasSortingKey())
return sorting_key.expression->getRequiredColumns();
return metadata.sorting_key.expression->getRequiredColumns();
return {};
}
Names IStorage::getSortingKeyColumns() const
{
if (hasSortingKey())
return sorting_key.column_names;
return metadata.sorting_key.column_names;
return {};
}
const StorageMetadataKeyField & IStorage::getPrimaryKey() const
const KeyDescription & IStorage::getPrimaryKey() const
{
return primary_key;
return metadata.primary_key;
}
void IStorage::setPrimaryKey(const StorageMetadataKeyField & primary_key_)
void IStorage::setPrimaryKey(const KeyDescription & primary_key_)
{
primary_key = primary_key_;
metadata.primary_key = primary_key_;
}
bool IStorage::isPrimaryKeyDefined() const
{
return primary_key.definition_ast != nullptr;
return metadata.primary_key.definition_ast != nullptr;
}
bool IStorage::hasPrimaryKey() const
{
return !primary_key.column_names.empty();
return !metadata.primary_key.column_names.empty();
}
Names IStorage::getColumnsRequiredForPrimaryKey() const
{
if (hasPrimaryKey())
return primary_key.expression->getRequiredColumns();
return metadata.primary_key.expression->getRequiredColumns();
return {};
}
Names IStorage::getPrimaryKeyColumns() const
{
if (hasSortingKey())
return primary_key.column_names;
if (!metadata.primary_key.column_names.empty())
return metadata.primary_key.column_names;
return {};
}
const StorageMetadataKeyField & IStorage::getSamplingKey() const
const KeyDescription & IStorage::getSamplingKey() const
{
return sampling_key;
return metadata.sampling_key;
}
void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_)
void IStorage::setSamplingKey(const KeyDescription & sampling_key_)
{
sampling_key = sampling_key_;
metadata.sampling_key = sampling_key_;
}
bool IStorage::isSamplingKeyDefined() const
{
return sampling_key.definition_ast != nullptr;
return metadata.sampling_key.definition_ast != nullptr;
}
bool IStorage::hasSamplingKey() const
{
return !sampling_key.column_names.empty();
return !metadata.sampling_key.column_names.empty();
}
Names IStorage::getColumnsRequiredForSampling() const
{
if (hasSamplingKey())
return sampling_key.expression->getRequiredColumns();
return metadata.sampling_key.expression->getRequiredColumns();
return {};
}
const TTLTableDescription & IStorage::getTableTTLs() const
TTLTableDescription IStorage::getTableTTLs() const
{
return table_ttl;
std::lock_guard lock(ttl_mutex);
return metadata.table_ttl;
}
void IStorage::setTableTTLs(const TTLTableDescription & table_ttl_)
{
table_ttl = table_ttl_;
std::lock_guard lock(ttl_mutex);
metadata.table_ttl = table_ttl_;
}
bool IStorage::hasAnyTableTTL() const
@ -559,39 +555,46 @@ bool IStorage::hasAnyTableTTL() const
return hasAnyMoveTTL() || hasRowsTTL();
}
const TTLColumnsDescription & IStorage::getColumnTTLs() const
TTLColumnsDescription IStorage::getColumnTTLs() const
{
return column_ttls_by_name;
std::lock_guard lock(ttl_mutex);
return metadata.column_ttls_by_name;
}
void IStorage::setColumnTTLs(const TTLColumnsDescription & column_ttls_by_name_)
{
column_ttls_by_name = column_ttls_by_name_;
std::lock_guard lock(ttl_mutex);
metadata.column_ttls_by_name = column_ttls_by_name_;
}
bool IStorage::hasAnyColumnTTL() const
{
return !column_ttls_by_name.empty();
std::lock_guard lock(ttl_mutex);
return !metadata.column_ttls_by_name.empty();
}
const TTLDescription & IStorage::getRowsTTL() const
TTLDescription IStorage::getRowsTTL() const
{
return table_ttl.rows_ttl;
std::lock_guard lock(ttl_mutex);
return metadata.table_ttl.rows_ttl;
}
bool IStorage::hasRowsTTL() const
{
return table_ttl.rows_ttl.expression != nullptr;
std::lock_guard lock(ttl_mutex);
return metadata.table_ttl.rows_ttl.expression != nullptr;
}
const TTLDescriptions & IStorage::getMoveTTLs() const
TTLDescriptions IStorage::getMoveTTLs() const
{
return table_ttl.move_ttl;
std::lock_guard lock(ttl_mutex);
return metadata.table_ttl.move_ttl;
}
bool IStorage::hasAnyMoveTTL() const
{
return !table_ttl.move_ttl.empty();
std::lock_guard lock(ttl_mutex);
return !metadata.table_ttl.move_ttl.empty();
}
@ -626,7 +629,8 @@ ColumnDependencies IStorage::getColumnDependencies(const NameSet & updated_colum
if (hasRowsTTL())
{
if (add_dependent_columns(getRowsTTL().expression, required_ttl_columns))
auto rows_expression = getRowsTTL().expression;
if (add_dependent_columns(rows_expression, required_ttl_columns))
{
/// Filter all columns, if rows TTL expression have to be recalculated.
for (const auto & column : getColumns().getAllPhysical())
@ -654,4 +658,34 @@ ColumnDependencies IStorage::getColumnDependencies(const NameSet & updated_colum
}
ASTPtr IStorage::getSettingsChanges() const
{
if (metadata.settings_changes)
return metadata.settings_changes->clone();
return nullptr;
}
void IStorage::setSettingsChanges(const ASTPtr & settings_changes_)
{
if (settings_changes_)
metadata.settings_changes = settings_changes_->clone();
else
metadata.settings_changes = nullptr;
}
const SelectQueryDescription & IStorage::getSelectQuery() const
{
return metadata.select;
}
void IStorage::setSelectQuery(const SelectQueryDescription & select_)
{
metadata.select = select_;
}
bool IStorage::hasSelectQuery() const
{
return metadata.select.select_query != nullptr;
}
}

69
src/Storages/IStorage.h
View File

@ -10,12 +10,9 @@
#include <Storages/SelectQueryInfo.h>
#include <Storages/TableStructureLockHolder.h>
#include <Storages/CheckResults.h>
#include <Storages/ColumnsDescription.h>
#include <Storages/IndicesDescription.h>
#include <Storages/ConstraintsDescription.h>
#include <Storages/StorageInMemoryMetadata.h>
#include <Storages/TTLDescription.h>
#include <Storages/ColumnDependency.h>
#include <Storages/SelectQueryDescription.h>
#include <Common/ActionLock.h>
#include <Common/Exception.h>
#include <Common/RWLock.h>
@ -141,6 +138,7 @@ public:
virtual ColumnSizeByName getColumnSizes() const { return {}; }
public: /// thread-unsafe part. lockStructure must be acquired
const ColumnsDescription & getColumns() const; /// returns combined set of columns
void setColumns(ColumnsDescription columns_); /// sets only real columns, possibly overwrites virtual ones.
@ -152,9 +150,17 @@ public: /// thread-unsafe part. lockStructure must be acquired
const ConstraintsDescription & getConstraints() const;
void setConstraints(ConstraintsDescription constraints_);
/// Returns storage metadata copy. Direct modification of
/// result structure doesn't affect storage.
virtual StorageInMemoryMetadata getInMemoryMetadata() const;
/// Storage settings
ASTPtr getSettingsChanges() const;
void setSettingsChanges(const ASTPtr & settings_changes_);
bool hasSettingsChanges() const { return metadata.settings_changes != nullptr; }
/// Select query for *View storages.
const SelectQueryDescription & getSelectQuery() const;
void setSelectQuery(const SelectQueryDescription & select_);
bool hasSelectQuery() const;
StorageInMemoryMetadata getInMemoryMetadata() const { return metadata; }
Block getSampleBlock() const; /// ordinary + materialized.
Block getSampleBlockWithVirtuals() const; /// ordinary + materialized + virtuals.
@ -199,18 +205,9 @@ private:
StorageID storage_id;
mutable std::mutex id_mutex;
ColumnsDescription columns;
IndicesDescription secondary_indices;
ConstraintsDescription constraints;
StorageMetadataKeyField partition_key;
StorageMetadataKeyField primary_key;
StorageMetadataKeyField sorting_key;
StorageMetadataKeyField sampling_key;
TTLColumnsDescription column_ttls_by_name;
TTLTableDescription table_ttl;
/// TODO (alesap) just use multiversion for atomic metadata
mutable std::mutex ttl_mutex;
StorageInMemoryMetadata metadata;
private:
RWLockImpl::LockHolder tryLockTimed(
const RWLock & rwlock, RWLockImpl::Type type, const String & query_id, const SettingSeconds & acquire_timeout) const;
@ -364,7 +361,7 @@ public:
/** Checks that alter commands can be applied to storage. For example, columns can be modified,
* or primary key can be changes, etc.
*/
virtual void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings);
virtual void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const;
/** ALTER tables with regard to its partitions.
* Should handle locks for each command on its own.
@ -443,12 +440,12 @@ public:
virtual Strings getDataPaths() const { return {}; }
/// Returns structure with partition key.
const StorageMetadataKeyField & getPartitionKey() const;
const KeyDescription & getPartitionKey() const;
/// Set partition key for storage (methods bellow, are just wrappers for this
/// struct).
void setPartitionKey(const StorageMetadataKeyField & partition_key_);
void setPartitionKey(const KeyDescription & partition_key_);
/// Returns ASTExpressionList of partition key expression for storage or nullptr if there is none.
ASTPtr getPartitionKeyAST() const { return partition_key.definition_ast; }
ASTPtr getPartitionKeyAST() const { return metadata.partition_key.definition_ast; }
/// Storage has user-defined (in CREATE query) partition key.
bool isPartitionKeyDefined() const;
/// Storage has partition key.
@ -458,12 +455,12 @@ public:
/// Returns structure with sorting key.
const StorageMetadataKeyField & getSortingKey() const;
const KeyDescription & getSortingKey() const;
/// Set sorting key for storage (methods bellow, are just wrappers for this
/// struct).
void setSortingKey(const StorageMetadataKeyField & sorting_key_);
void setSortingKey(const KeyDescription & sorting_key_);
/// Returns ASTExpressionList of sorting key expression for storage or nullptr if there is none.
ASTPtr getSortingKeyAST() const { return sorting_key.definition_ast; }
ASTPtr getSortingKeyAST() const { return metadata.sorting_key.definition_ast; }
/// Storage has user-defined (in CREATE query) sorting key.
bool isSortingKeyDefined() const;
/// Storage has sorting key. It means, that it contains at least one column.
@ -475,12 +472,12 @@ public:
Names getSortingKeyColumns() const;
/// Returns structure with primary key.
const StorageMetadataKeyField & getPrimaryKey() const;
const KeyDescription & getPrimaryKey() const;
/// Set primary key for storage (methods bellow, are just wrappers for this
/// struct).
void setPrimaryKey(const StorageMetadataKeyField & primary_key_);
void setPrimaryKey(const KeyDescription & primary_key_);
/// Returns ASTExpressionList of primary key expression for storage or nullptr if there is none.
ASTPtr getPrimaryKeyAST() const { return primary_key.definition_ast; }
ASTPtr getPrimaryKeyAST() const { return metadata.primary_key.definition_ast; }
/// Storage has user-defined (in CREATE query) sorting key.
bool isPrimaryKeyDefined() const;
/// Storage has primary key (maybe part of some other key). It means, that
@ -493,12 +490,12 @@ public:
Names getPrimaryKeyColumns() const;
/// Returns structure with sampling key.
const StorageMetadataKeyField & getSamplingKey() const;
const KeyDescription & getSamplingKey() const;
/// Set sampling key for storage (methods bellow, are just wrappers for this
/// struct).
void setSamplingKey(const StorageMetadataKeyField & sampling_key_);
void setSamplingKey(const KeyDescription & sampling_key_);
/// Returns sampling expression AST for storage or nullptr if there is none.
ASTPtr getSamplingKeyAST() const { return sampling_key.definition_ast; }
ASTPtr getSamplingKeyAST() const { return metadata.sampling_key.definition_ast; }
/// Storage has user-defined (in CREATE query) sampling key.
bool isSamplingKeyDefined() const;
/// Storage has sampling key.
@ -517,22 +514,22 @@ public:
virtual StoragePolicyPtr getStoragePolicy() const { return {}; }
/// Common tables TTLs (for rows and moves).
const TTLTableDescription & getTableTTLs() const;
TTLTableDescription getTableTTLs() const;
void setTableTTLs(const TTLTableDescription & table_ttl_);
bool hasAnyTableTTL() const;
/// Separate TTLs for columns.
const TTLColumnsDescription & getColumnTTLs() const;
TTLColumnsDescription getColumnTTLs() const;
void setColumnTTLs(const TTLColumnsDescription & column_ttls_by_name_);
bool hasAnyColumnTTL() const;
/// Just wrapper for table TTLs, return rows part of table TTLs.
const TTLDescription & getRowsTTL() const;
TTLDescription getRowsTTL() const;
bool hasRowsTTL() const;
/// Just wrapper for table TTLs, return moves (to disks or volumes) parts of
/// table TTL.
const TTLDescriptions & getMoveTTLs() const;
TTLDescriptions getMoveTTLs() const;
bool hasAnyMoveTTL() const;
/// If it is possible to quickly determine exact number of rows in the table at this moment of time, then return it.

46
src/Storages/IndicesDescription.cpp
View File

@ -19,6 +19,52 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
};
IndexDescription::IndexDescription(const IndexDescription & other)
: definition_ast(other.definition_ast ? other.definition_ast->clone() : nullptr)
, expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr)
, name(other.name)
, type(other.type)
, arguments(other.arguments)
, column_names(other.column_names)
, data_types(other.data_types)
, sample_block(other.sample_block)
, granularity(other.granularity)
{
if (other.expression)
expression = std::make_shared<ExpressionActions>(*other.expression);
}
IndexDescription & IndexDescription::operator=(const IndexDescription & other)
{
if (&other == this)
return *this;
if (other.definition_ast)
definition_ast = other.definition_ast->clone();
else
definition_ast.reset();
if (other.expression_list_ast)
expression_list_ast = other.expression_list_ast->clone();
else
expression_list_ast.reset();
name = other.name;
type = other.type;
if (other.expression)
expression = std::make_shared<ExpressionActions>(*other.expression);
else
expression.reset();
arguments = other.arguments;
column_names = other.column_names;
data_types = other.data_types;
sample_block = other.sample_block;
granularity = other.granularity;
return *this;
}
IndexDescription IndexDescription::getIndexFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context)
{

7
src/Storages/IndicesDescription.h
View File

@ -48,6 +48,13 @@ struct IndexDescription
/// Parse index from definition AST
static IndexDescription getIndexFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context);
IndexDescription() = default;
/// We need custom copy constructors because we don't want
/// unintentionaly share AST variables and modify them.
IndexDescription(const IndexDescription & other);
IndexDescription & operator=(const IndexDescription & other);
};
/// All secondary indices in storage

123
src/Storages/KeyDescription.cpp Normal file
View File

@ -0,0 +1,123 @@
#include <Storages/KeyDescription.h>
#include <Functions/IFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <Interpreters/SyntaxAnalyzer.h>
#include <Storages/extractKeyExpressionList.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
KeyDescription::KeyDescription(const KeyDescription & other)
: definition_ast(other.definition_ast ? other.definition_ast->clone() : nullptr)
, expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr)
, sample_block(other.sample_block)
, column_names(other.column_names)
, data_types(other.data_types)
, additional_column(other.additional_column)
{
if (other.expression)
expression = std::make_shared<ExpressionActions>(*other.expression);
}
KeyDescription & KeyDescription::operator=(const KeyDescription & other)
{
if (&other == this)
return *this;
if (other.definition_ast)
definition_ast = other.definition_ast->clone();
else
definition_ast.reset();
if (other.expression_list_ast)
expression_list_ast = other.expression_list_ast->clone();
else
expression_list_ast.reset();
if (other.expression)
expression = std::make_shared<ExpressionActions>(*other.expression);
else
expression.reset();
sample_block = other.sample_block;
column_names = other.column_names;
data_types = other.data_types;
/// additional_column is constant property It should never be lost.
if (additional_column.has_value() && !other.additional_column.has_value())
throw Exception("Wrong key assignment, loosing additional_column", ErrorCodes::LOGICAL_ERROR);
additional_column = other.additional_column;
return *this;
}
void KeyDescription::recalculateWithNewAST(
const ASTPtr & new_ast,
const ColumnsDescription & columns,
const Context & context)
{
*this = getSortingKeyFromAST(new_ast, columns, context, additional_column);
}
void KeyDescription::recalculateWithNewColumns(
const ColumnsDescription & new_columns,
const Context & context)
{
*this = getSortingKeyFromAST(definition_ast, new_columns, context, additional_column);
}
KeyDescription KeyDescription::getKeyFromAST(
const ASTPtr & definition_ast,
const ColumnsDescription & columns,
const Context & context)
{
return getSortingKeyFromAST(definition_ast, columns, context, {});
}
KeyDescription KeyDescription::getSortingKeyFromAST(
const ASTPtr & definition_ast,
const ColumnsDescription & columns,
const Context & context,
const std::optional<String> & additional_column)
{
KeyDescription result;
result.definition_ast = definition_ast;
result.expression_list_ast = extractKeyExpressionList(definition_ast);
if (additional_column)
{
result.additional_column = additional_column;
ASTPtr column_identifier = std::make_shared<ASTIdentifier>(*additional_column);
result.expression_list_ast->children.push_back(column_identifier);
}
const auto & children = result.expression_list_ast->children;
for (const auto & child : children)
result.column_names.emplace_back(child->getColumnName());
{
auto expr = result.expression_list_ast->clone();
auto syntax_result = SyntaxAnalyzer(context).analyze(expr, columns.getAllPhysical());
/// In expression we also need to store source columns
result.expression = ExpressionAnalyzer(expr, syntax_result, context).getActions(false);
/// In sample block we use just key columns
result.sample_block = ExpressionAnalyzer(expr, syntax_result, context).getActions(true)->getSampleBlock();
}
for (size_t i = 0; i < result.sample_block.columns(); ++i)
result.data_types.emplace_back(result.sample_block.getByPosition(i).type);
return result;
}
}

74
src/Storages/KeyDescription.h Normal file
View File

@ -0,0 +1,74 @@
#pragma once
#include <Interpreters/ExpressionActions.h>
#include <Parsers/IAST_fwd.h>
#include <Storages/ColumnsDescription.h>
namespace DB
{
/// Common structure for primary, partition and other storage keys
struct KeyDescription
{
/// User defined AST in CREATE/ALTER query. This field may be empty, but key
/// can exists because some of them maybe set implicitly (for example,
/// primary key in merge tree can be part of sorting key)
ASTPtr definition_ast;
/// ASTExpressionList with key fields, example: (x, toStartOfMonth(date))).
ASTPtr expression_list_ast;
/// Expression from expression_list_ast created by ExpressionAnalyzer. Useful,
/// when you need to get required columns for key, example: a, date, b.
ExpressionActionsPtr expression;
/// Sample block with key columns (names, types, empty column)
Block sample_block;
/// Column names in key definition, example: x, toStartOfMonth(date), a * b.
Names column_names;
/// Types from sample block ordered in columns order.
DataTypes data_types;
/// Additional key column added by storage type. Never changes after
/// initialization with non empty value. Doesn't stored in definition_ast,
/// but added to expression_list_ast and all its derivatives.
std::optional<String> additional_column;
/// Parse key structure from key definition. Requires all columns, available
/// in storage.
static KeyDescription getKeyFromAST(
const ASTPtr & definition_ast,
const ColumnsDescription & columns,
const Context & context);
/// Sorting key can contain additional column defined by storage type (like
/// Version column in VersionedCollapsingMergeTree).
static KeyDescription getSortingKeyFromAST(
const ASTPtr & definition_ast,
const ColumnsDescription & columns,
const Context & context,
const std::optional<String> & additional_column);
/// Recalculate all expressions and fields for key with new columns without
/// changes in constant fields. Just wrapper for static methods.
void recalculateWithNewColumns(
const ColumnsDescription & new_columns,
const Context & context);
/// Recalculate all expressions and fields for key with new ast without
/// changes in constant fields. Just wrapper for static methods.
void recalculateWithNewAST(
const ASTPtr & new_ast,
const ColumnsDescription & columns,
const Context & context);
KeyDescription() = default;
/// We need custom copy constructors because we don't want
/// unintentionaly share AST variables and modify them.
KeyDescription(const KeyDescription & other);
KeyDescription & operator=(const KeyDescription & other);
};
}

1
src/Storages/LiveView/StorageLiveView.h
View File

@ -165,6 +165,7 @@ public:
const Context & context);
private:
/// TODO move to common struct SelectQueryDescription
StorageID select_table_id = StorageID::createEmpty(); /// Will be initialized in constructor
ASTPtr inner_query; /// stored query : SELECT * FROM ( SELECT a FROM A)
ASTPtr inner_subquery; /// stored query's innermost subquery if any

290
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -106,7 +106,6 @@ namespace ErrorCodes
extern const int READONLY_SETTING;
extern const int ABORTED;
extern const int UNKNOWN_PART_TYPE;
extern const int UNEXPECTED_AST_STRUCTURE;
extern const int UNKNOWN_DISK;
extern const int NOT_ENOUGH_SPACE;
extern const int ALTER_OF_COLUMN_IS_FORBIDDEN;
@ -119,7 +118,7 @@ const char * DELETE_ON_DESTROY_MARKER_PATH = "delete-on-destroy.txt";
MergeTreeData::MergeTreeData(
const StorageID & table_id_,
const String & relative_data_path_,
const StorageInMemoryMetadata & metadata,
const StorageInMemoryMetadata & metadata_,
Context & context_,
const String & date_column_name,
const MergingParams & merging_params_,
@ -130,7 +129,6 @@ MergeTreeData::MergeTreeData(
: IStorage(table_id_)
, global_context(context_)
, merging_params(merging_params_)
, settings_ast(metadata.settings_ast)
, require_part_metadata(require_part_metadata_)
, relative_data_path(relative_data_path_)
, broken_part_callback(broken_part_callback_)
@ -144,23 +142,21 @@ MergeTreeData::MergeTreeData(
if (relative_data_path.empty())
throw Exception("MergeTree storages require data path", ErrorCodes::INCORRECT_FILE_NAME);
setSettingsChanges(metadata_.settings_changes);
const auto settings = getSettings();
setProperties(metadata, /*only_check*/ false, attach);
setProperties(metadata_, attach);
/// NOTE: using the same columns list as is read when performing actual merges.
merging_params.check(getColumns().getAllPhysical());
if (metadata.sample_by_ast != nullptr)
if (metadata_.sampling_key.definition_ast != nullptr)
{
StorageMetadataKeyField candidate_sampling_key = StorageMetadataKeyField::getKeyFromAST(
metadata.sample_by_ast, getColumns(), global_context);
const auto & pk_sample_block = getPrimaryKey().sample_block;
if (!pk_sample_block.has(candidate_sampling_key.column_names[0]) && !attach
if (!pk_sample_block.has(metadata_.sampling_key.column_names[0]) && !attach
&& !settings->compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility.
throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS);
setSamplingKey(candidate_sampling_key);
setSamplingKey(metadata_.sampling_key);
}
MergeTreeDataFormatVersion min_format_version(0);
@ -169,7 +165,8 @@ MergeTreeData::MergeTreeData(
try
{
auto partition_by_ast = makeASTFunction("toYYYYMM", std::make_shared<ASTIdentifier>(date_column_name));
initPartitionKey(partition_by_ast);
auto partition_key = KeyDescription::getKeyFromAST(partition_by_ast, getColumns(), global_context);
initPartitionKey(partition_key);
if (minmax_idx_date_column_pos == -1)
throw Exception("Could not find Date column", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -184,11 +181,11 @@ MergeTreeData::MergeTreeData(
else
{
is_custom_partitioned = true;
initPartitionKey(metadata.partition_by_ast);
initPartitionKey(metadata_.partition_key);
min_format_version = MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING;
}
setTTLExpressions(metadata.columns, metadata.ttl_for_table_ast);
setTTLExpressions(metadata_);
/// format_file always contained on any data path
PathWithDisk version_file;
@ -245,32 +242,6 @@ MergeTreeData::MergeTreeData(
LOG_WARNING(log, "{} Settings 'min_bytes_for_wide_part' and 'min_bytes_for_wide_part' will be ignored.", reason);
}
StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const
{
StorageInMemoryMetadata metadata(getColumns(), getSecondaryIndices(), getConstraints());
if (isPartitionKeyDefined())
metadata.partition_by_ast = getPartitionKeyAST()->clone();
if (isSortingKeyDefined())
metadata.order_by_ast = getSortingKeyAST()->clone();
if (isPrimaryKeyDefined())
metadata.primary_key_ast = getPrimaryKeyAST()->clone();
if (hasAnyTableTTL())
metadata.ttl_for_table_ast = getTableTTLs().definition_ast->clone();
if (isSamplingKeyDefined())
metadata.sample_by_ast = getSamplingKeyAST()->clone();
if (settings_ast)
metadata.settings_ast = settings_ast->clone();
return metadata;
}
StoragePolicyPtr MergeTreeData::getStoragePolicy() const
{
return global_context.getStoragePolicy(getSettings()->storage_policy);
@ -304,37 +275,30 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam
}
}
void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool only_check, bool attach)
void MergeTreeData::checkProperties(const StorageInMemoryMetadata & new_metadata, bool attach) const
{
if (!metadata.order_by_ast)
if (!new_metadata.sorting_key.definition_ast)
throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS);
ASTPtr new_sorting_key_expr_list = extractKeyExpressionList(metadata.order_by_ast);
ASTPtr new_primary_key_expr_list = metadata.primary_key_ast
? extractKeyExpressionList(metadata.primary_key_ast) : new_sorting_key_expr_list->clone();
KeyDescription new_sorting_key = new_metadata.sorting_key;
KeyDescription new_primary_key = new_metadata.primary_key;
if (merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing)
new_sorting_key_expr_list->children.push_back(std::make_shared<ASTIdentifier>(merging_params.version_column));
size_t primary_key_size = new_primary_key_expr_list->children.size();
size_t sorting_key_size = new_sorting_key_expr_list->children.size();
size_t sorting_key_size = new_sorting_key.column_names.size();
size_t primary_key_size = new_primary_key.column_names.size();
if (primary_key_size > sorting_key_size)
throw Exception("Primary key must be a prefix of the sorting key, but its length: "
+ toString(primary_key_size) + " is greater than the sorting key length: " + toString(sorting_key_size),
ErrorCodes::BAD_ARGUMENTS);
Names new_primary_key_columns;
Names new_sorting_key_columns;
NameSet primary_key_columns_set;
for (size_t i = 0; i < sorting_key_size; ++i)
{
String sorting_key_column = new_sorting_key_expr_list->children[i]->getColumnName();
new_sorting_key_columns.push_back(sorting_key_column);
const String & sorting_key_column = new_sorting_key.column_names[i];
if (i < primary_key_size)
{
String pk_column = new_primary_key_expr_list->children[i]->getColumnName();
const String & pk_column = new_primary_key.column_names[i];
if (pk_column != sorting_key_column)
throw Exception("Primary key must be a prefix of the sorting key, but in position "
+ toString(i) + " its column is " + pk_column + ", not " + sorting_key_column,
@ -343,18 +307,20 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool
if (!primary_key_columns_set.emplace(pk_column).second)
throw Exception("Primary key contains duplicate columns", ErrorCodes::BAD_ARGUMENTS);
new_primary_key_columns.push_back(pk_column);
}
}
auto all_columns = metadata.columns.getAllPhysical();
auto all_columns = new_metadata.columns.getAllPhysical();
/// Order by check AST
if (hasSortingKey() && only_check)
if (hasSortingKey())
{
/// This is ALTER, not CREATE/ATTACH TABLE. Let us check that all new columns used in the sorting key
/// expression have just been added (so that the sorting order is guaranteed to be valid with the new key).
Names new_primary_key_columns = new_primary_key.column_names;
Names new_sorting_key_columns = new_sorting_key.column_names;
ASTPtr added_key_column_expr_list = std::make_shared<ASTExpressionList>();
const auto & old_sorting_key_columns = getSortingKeyColumns();
for (size_t new_i = 0, old_i = 0; new_i < sorting_key_size; ++new_i)
@ -362,12 +328,12 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool
if (old_i < old_sorting_key_columns.size())
{
if (new_sorting_key_columns[new_i] != old_sorting_key_columns[old_i])
added_key_column_expr_list->children.push_back(new_sorting_key_expr_list->children[new_i]);
added_key_column_expr_list->children.push_back(new_sorting_key.expression_list_ast->children[new_i]);
else
++old_i;
}
else
added_key_column_expr_list->children.push_back(new_sorting_key_expr_list->children[new_i]);
added_key_column_expr_list->children.push_back(new_sorting_key.expression_list_ast->children[new_i]);
}
if (!added_key_column_expr_list->children.empty())
@ -386,7 +352,7 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool
"added to the sorting key. You can add expressions that use only the newly added columns",
ErrorCodes::BAD_ARGUMENTS);
if (metadata.columns.getDefaults().count(col))
if (new_metadata.columns.getDefaults().count(col))
throw Exception("Newly added column " + col + " has a default expression, so adding "
"expressions that use it to the sorting key is forbidden",
ErrorCodes::BAD_ARGUMENTS);
@ -394,39 +360,11 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool
}
}
auto new_sorting_key_syntax = SyntaxAnalyzer(global_context).analyze(new_sorting_key_expr_list, all_columns);
auto new_sorting_key_expr = ExpressionAnalyzer(new_sorting_key_expr_list, new_sorting_key_syntax, global_context)
.getActions(false);
auto new_sorting_key_sample =
ExpressionAnalyzer(new_sorting_key_expr_list, new_sorting_key_syntax, global_context)
.getActions(true)->getSampleBlock();
checkKeyExpression(*new_sorting_key_expr, new_sorting_key_sample, "Sorting");
auto new_primary_key_syntax = SyntaxAnalyzer(global_context).analyze(new_primary_key_expr_list, all_columns);
auto new_primary_key_expr = ExpressionAnalyzer(new_primary_key_expr_list, new_primary_key_syntax, global_context)
.getActions(false);
Block new_primary_key_sample;
DataTypes new_primary_key_data_types;
for (size_t i = 0; i < primary_key_size; ++i)
if (!new_metadata.secondary_indices.empty())
{
const auto & elem = new_sorting_key_sample.getByPosition(i);
new_primary_key_sample.insert(elem);
new_primary_key_data_types.push_back(elem.type);
}
std::unordered_set<String> indices_names;
DataTypes new_sorting_key_data_types;
for (size_t i = 0; i < sorting_key_size; ++i)
{
new_sorting_key_data_types.push_back(new_sorting_key_sample.getByPosition(i).type);
}
if (!metadata.secondary_indices.empty())
{
std::set<String> indices_names;
for (const auto & index : metadata.secondary_indices)
for (const auto & index : new_metadata.secondary_indices)
{
MergeTreeIndexFactory::instance().validate(index, attach);
@ -440,39 +378,27 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool
}
}
if (!only_check)
{
setColumns(std::move(metadata.columns));
checkKeyExpression(*new_sorting_key.expression, new_sorting_key.sample_block, "Sorting");
StorageMetadataKeyField new_sorting_key;
new_sorting_key.definition_ast = metadata.order_by_ast;
new_sorting_key.column_names = std::move(new_sorting_key_columns);
new_sorting_key.expression_list_ast = std::move(new_sorting_key_expr_list);
new_sorting_key.expression = std::move(new_sorting_key_expr);
new_sorting_key.sample_block = std::move(new_sorting_key_sample);
new_sorting_key.data_types = std::move(new_sorting_key_data_types);
setSortingKey(new_sorting_key);
}
StorageMetadataKeyField new_primary_key;
new_primary_key.definition_ast = metadata.primary_key_ast;
new_primary_key.column_names = std::move(new_primary_key_columns);
new_primary_key.expression_list_ast = std::move(new_primary_key_expr_list);
new_primary_key.expression = std::move(new_primary_key_expr);
new_primary_key.sample_block = std::move(new_primary_key_sample);
new_primary_key.data_types = std::move(new_primary_key_data_types);
setPrimaryKey(new_primary_key);
void MergeTreeData::setProperties(const StorageInMemoryMetadata & new_metadata, bool attach)
{
checkProperties(new_metadata, attach);
setSecondaryIndices(metadata.secondary_indices);
setConstraints(metadata.constraints);
}
/// Other parts of metadata initialized is separate methods
setColumns(std::move(new_metadata.columns));
setSecondaryIndices(std::move(new_metadata.secondary_indices));
setConstraints(std::move(new_metadata.constraints));
setSortingKey(std::move(new_metadata.sorting_key));
setPrimaryKey(std::move(new_metadata.primary_key));
}
namespace
{
ExpressionActionsPtr getCombinedIndicesExpression(
const StorageMetadataKeyField & key,
const KeyDescription & key,
const IndicesDescription & indices,
const ColumnsDescription & columns,
const Context & context)
@ -499,32 +425,9 @@ ExpressionActionsPtr MergeTreeData::getSortingKeyAndSkipIndicesExpression() cons
return getCombinedIndicesExpression(getSortingKey(), getSecondaryIndices(), getColumns(), global_context);
}
ASTPtr MergeTreeData::extractKeyExpressionList(const ASTPtr & node)
void MergeTreeData::initPartitionKey(const KeyDescription & new_partition_key)
{
if (!node)
return std::make_shared<ASTExpressionList>();
const auto * expr_func = node->as<ASTFunction>();
if (expr_func && expr_func->name == "tuple")
{
/// Primary key is specified in tuple, extract its arguments.
return expr_func->arguments->clone();
}
else
{
/// Primary key consists of one column.
auto res = std::make_shared<ASTExpressionList>();
res->children.push_back(node);
return res;
}
}
void MergeTreeData::initPartitionKey(ASTPtr partition_by_ast)
{
StorageMetadataKeyField new_partition_key = StorageMetadataKeyField::getKeyFromAST(partition_by_ast, getColumns(), global_context);
if (new_partition_key.expression_list_ast->children.empty())
return;
@ -580,15 +483,11 @@ void MergeTreeData::initPartitionKey(ASTPtr partition_by_ast)
}
void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
const ASTPtr & new_ttl_table_ast, bool only_check)
void MergeTreeData::checkTTLExpressions(const StorageInMemoryMetadata & new_metadata) const
{
auto new_column_ttls = new_metadata.column_ttls_by_name;
auto new_column_ttls_asts = new_columns.getColumnTTLs();
TTLColumnsDescription new_column_ttl_by_name = getColumnTTLs();
if (!new_column_ttls_asts.empty())
if (!new_column_ttls.empty())
{
NameSet columns_ttl_forbidden;
@ -600,76 +499,39 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
for (const auto & col : getColumnsRequiredForSortingKey())
columns_ttl_forbidden.insert(col);
for (const auto & [name, ast] : new_column_ttls_asts)
for (const auto & [name, ttl_description] : new_column_ttls)
{
if (columns_ttl_forbidden.count(name))
throw Exception("Trying to set TTL for key column " + name, ErrorCodes::ILLEGAL_COLUMN);
else
{
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_columns, global_context, getPrimaryKey());
new_column_ttl_by_name[name] = new_ttl_entry;
}
}
if (!only_check)
setColumnTTLs(new_column_ttl_by_name);
}
auto new_table_ttl = new_metadata.table_ttl;
if (new_ttl_table_ast)
if (new_table_ttl.definition_ast)
{
TTLDescriptions update_move_ttl_entries;
TTLDescription update_rows_ttl_entry;
bool seen_delete_ttl = false;
for (const auto & ttl_element_ptr : new_ttl_table_ast->children)
for (const auto & move_ttl : new_table_ttl.move_ttl)
{
const auto * ttl_element = ttl_element_ptr->as<ASTTTLElement>();
if (!ttl_element)
throw Exception("Unexpected AST element in TTL expression", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
if (ttl_element->destination_type == DataDestinationType::DELETE)
if (!getDestinationForTTL(move_ttl))
{
if (seen_delete_ttl)
{
throw Exception("More than one DELETE TTL expression is not allowed", ErrorCodes::BAD_TTL_EXPRESSION);
}
update_rows_ttl_entry = TTLDescription::getTTLFromAST(ttl_element_ptr, new_columns, global_context, getPrimaryKey());
seen_delete_ttl = true;
String message;
if (move_ttl.destination_type == DataDestinationType::DISK)
message = "No such disk " + backQuote(move_ttl.destination_name) + " for given storage policy.";
else
message = "No such volume " + backQuote(move_ttl.destination_name) + " for given storage policy.";
throw Exception(message, ErrorCodes::BAD_TTL_EXPRESSION);
}
else
{
auto new_ttl_entry = TTLDescription::getTTLFromAST(ttl_element_ptr, new_columns, global_context, getPrimaryKey());
if (!getDestinationForTTL(new_ttl_entry))
{
String message;
if (new_ttl_entry.destination_type == DataDestinationType::DISK)
message = "No such disk " + backQuote(new_ttl_entry.destination_name) + " for given storage policy.";
else
message = "No such volume " + backQuote(new_ttl_entry.destination_name) + " for given storage policy.";
throw Exception(message, ErrorCodes::BAD_TTL_EXPRESSION);
}
update_move_ttl_entries.emplace_back(std::move(new_ttl_entry));
}
}
if (!only_check)
{
TTLTableDescription new_table_ttl
{
.definition_ast = new_ttl_table_ast,
.rows_ttl = update_rows_ttl_entry,
.move_ttl = update_move_ttl_entries,
};
auto move_ttl_entries_lock = std::lock_guard<std::mutex>(move_ttl_entries_mutex);
setTableTTLs(new_table_ttl);
}
}
}
/// Todo replace columns with TTL for columns
void MergeTreeData::setTTLExpressions(const StorageInMemoryMetadata & new_metadata)
{
checkTTLExpressions(new_metadata);
setColumnTTLs(new_metadata.column_ttls_by_name);
setTableTTLs(new_metadata.table_ttl);
}
void MergeTreeData::checkStoragePolicy(const StoragePolicyPtr & new_storage_policy) const
{
@ -1379,12 +1241,12 @@ bool isMetadataOnlyConversion(const IDataType * from, const IDataType * to)
}
void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const Settings & settings)
void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const
{
/// Check that needed transformations can be applied to the list of columns without considering type conversions.
StorageInMemoryMetadata metadata = getInMemoryMetadata();
commands.apply(metadata, global_context);
if (getSecondaryIndices().empty() && !metadata.secondary_indices.empty() &&
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
commands.apply(new_metadata, global_context);
if (getSecondaryIndices().empty() && !new_metadata.secondary_indices.empty() &&
!settings.allow_experimental_data_skipping_indices)
throw Exception("You must set the setting `allow_experimental_data_skipping_indices` to 1 " \
"before using data skipping indices.", ErrorCodes::BAD_ARGUMENTS);
@ -1474,14 +1336,15 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S
}
}
setProperties(metadata, /* only_check = */ true);
checkProperties(new_metadata);
setTTLExpressions(metadata.columns, metadata.ttl_for_table_ast, /* only_check = */ true);
checkTTLExpressions(new_metadata);
if (settings_ast)
if (hasSettingsChanges())
{
const auto & current_changes = settings_ast->as<const ASTSetQuery &>().changes;
const auto & new_changes = metadata.settings_ast->as<const ASTSetQuery &>().changes;
const auto current_changes = getSettingsChanges()->as<const ASTSetQuery &>().changes;
const auto & new_changes = new_metadata.settings_changes->as<const ASTSetQuery &>().changes;
for (const auto & changed_setting : new_changes)
{
if (MergeTreeSettings::findIndex(changed_setting.name) == MergeTreeSettings::npos)
@ -1619,7 +1482,7 @@ void MergeTreeData::changeSettings(
MergeTreeSettings copy = *getSettings();
copy.applyChanges(new_changes);
storage_settings.set(std::make_unique<const MergeTreeSettings>(copy));
settings_ast = new_settings;
setSettingsChanges(new_settings);
}
}
@ -2943,7 +2806,6 @@ MergeTreeData::selectTTLEntryForTTLInfos(const IMergeTreeDataPart::TTLInfos & tt
time_t max_max_ttl = 0;
TTLDescriptions::const_iterator best_entry_it;
auto lock = std::lock_guard(move_ttl_entries_mutex);
const auto & move_ttl_entries = getMoveTTLs();
for (auto ttl_entry_it = move_ttl_entries.begin(); ttl_entry_it != move_ttl_entries.end(); ++ttl_entry_it)
{

36
src/Storages/MergeTree/MergeTreeData.h
View File

@ -21,6 +21,7 @@
#include <Interpreters/PartLog.h>
#include <Disks/StoragePolicy.h>
#include <Interpreters/Aggregator.h>
#include <Storages/extractKeyExpressionList.h>
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/ordered_index.hpp>
@ -323,7 +324,7 @@ public:
/// attach - whether the existing table is attached or the new table is created.
MergeTreeData(const StorageID & table_id_,
const String & relative_data_path_,
const StorageInMemoryMetadata & metadata,
const StorageInMemoryMetadata & metadata_,
Context & context_,
const String & date_column_name,
const MergingParams & merging_params_,
@ -333,9 +334,6 @@ public:
BrokenPartCallback broken_part_callback_ = [](const String &){});
/// See comments about methods below in IStorage interface
StorageInMemoryMetadata getInMemoryMetadata() const override;
StoragePolicyPtr getStoragePolicy() const override;
bool supportsPrewhere() const override { return true; }
@ -498,7 +496,7 @@ public:
/// - all type conversions can be done.
/// - columns corresponding to primary key, indices, sign, sampling expression and date are not affected.
/// If something is wrong, throws an exception.
void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) override;
void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const override;
/// Change MergeTreeSettings
void changeSettings(
@ -514,11 +512,12 @@ public:
broken_part_callback(name);
}
/** Get the key expression AST as an ASTExpressionList. It can be specified
* in the tuple: (CounterID, Date), or as one column: CounterID.
*/
static ASTPtr extractKeyExpressionList(const ASTPtr & node);
/// TODO (alesap) Duplicate method required for compatibility.
/// Must be removed.
static ASTPtr extractKeyExpressionList(const ASTPtr & node)
{
return DB::extractKeyExpressionList(node);
}
/// Check that the part is not broken and calculate the checksums for it if they are not present.
MutableDataPartPtr loadPartAndFixMetadata(const VolumePtr & volume, const String & relative_path) const;
@ -649,11 +648,6 @@ public:
std::optional<TTLDescription> selectTTLEntryForTTLInfos(const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const;
/// This mutex is required for background move operations which do not
/// obtain global locks.
/// TODO (alesap) It will be removed after metadata became atomic
mutable std::mutex move_ttl_entries_mutex;
/// Limiting parallel sends per one table, used in DataPartsExchange
std::atomic_uint current_table_sends {0};
@ -680,8 +674,6 @@ protected:
friend struct ReplicatedMergeTreeTableMetadata;
friend class StorageReplicatedMergeTree;
ASTPtr settings_ast;
bool require_part_metadata;
String relative_data_path;
@ -788,12 +780,14 @@ protected:
/// The same for clearOldTemporaryDirectories.
std::mutex clear_old_temporary_directories_mutex;
void setProperties(const StorageInMemoryMetadata & metadata, bool only_check = false, bool attach = false);
void checkProperties(const StorageInMemoryMetadata & new_metadata, bool attach = false) const;
void initPartitionKey(ASTPtr partition_by_ast);
void setProperties(const StorageInMemoryMetadata & new_metadata, bool attach = false);
void setTTLExpressions(const ColumnsDescription & columns,
const ASTPtr & new_ttl_table_ast, bool only_check = false);
void initPartitionKey(const KeyDescription & new_partition_key);
void checkTTLExpressions(const StorageInMemoryMetadata & new_metadata) const;
void setTTLExpressions(const StorageInMemoryMetadata & new_metadata);
void checkStoragePolicy(const StoragePolicyPtr & new_storage_policy) const;

5
src/Storages/MergeTree/StorageFromMergeTreeDataPart.h
View File

@ -39,11 +39,6 @@ public:
return part->storage.mayBenefitFromIndexForIn(left_in_operand, query_context);
}
StorageInMemoryMetadata getInMemoryMetadata() const override
{
return part->storage.getInMemoryMetadata();
}
NamesAndTypesList getVirtuals() const override
{
return part->storage.getVirtuals();

95
src/Storages/MergeTree/registerStorageMergeTree.cpp
View File

@ -417,6 +417,9 @@ static StoragePtr create(const StorageFactory::Arguments & args)
++arg_num;
}
/// This merging param maybe used as part of sorting key
std::optional<String> merging_param_key_arg;
if (merging_params.mode == MergeTreeData::MergingParams::Collapsing)
{
if (!tryGetIdentifierNameInto(engine_args[arg_cnt - 1], merging_params.sign_column))
@ -480,50 +483,83 @@ static StoragePtr create(const StorageFactory::Arguments & args)
ErrorCodes::BAD_ARGUMENTS);
--arg_cnt;
/// Version collapsing is the only engine which add additional column to
/// sorting key.
merging_param_key_arg = merging_params.version_column;
}
String date_column_name;
ASTPtr partition_by_ast;
ASTPtr order_by_ast;
ASTPtr primary_key_ast;
ASTPtr sample_by_ast;
ASTPtr ttl_table_ast;
ASTPtr settings_ast;
IndicesDescription indices_description;
ConstraintsDescription constraints_description;
StorageInMemoryMetadata metadata;
metadata.columns = args.columns;
std::unique_ptr<MergeTreeSettings> storage_settings = std::make_unique<MergeTreeSettings>(args.context.getMergeTreeSettings());
if (is_extended_storage_def)
{
ASTPtr partition_by_key;
if (args.storage_def->partition_by)
partition_by_ast = args.storage_def->partition_by->ptr();
partition_by_key = args.storage_def->partition_by->ptr();
/// Partition key may be undefined, but despite this we store it's empty
/// value in partition_key structure. MergeTree checks this case and use
/// single default partition with name "all".
metadata.partition_key = KeyDescription::getKeyFromAST(partition_by_key, metadata.columns, args.context);
if (!args.storage_def->order_by)
throw Exception("You must provide an ORDER BY expression in the table definition. "
"If you don't want this table to be sorted, use ORDER BY tuple()",
ErrorCodes::BAD_ARGUMENTS);
order_by_ast = args.storage_def->order_by->ptr();
/// Get sorting key from engine arguments.
///
/// NOTE: store merging_param_key_arg as additional key column. We do it
/// before storage creation. After that storage will just copy this
/// column if sorting key will be changed.
metadata.sorting_key = KeyDescription::getSortingKeyFromAST(args.storage_def->order_by->ptr(), metadata.columns, args.context, merging_param_key_arg);
/// If primary key explicitely defined, than get it from AST
if (args.storage_def->primary_key)
primary_key_ast = args.storage_def->primary_key->ptr();
{
metadata.primary_key = KeyDescription::getKeyFromAST(args.storage_def->primary_key->ptr(), metadata.columns, args.context);
}
else /// Otherwise we copy it from primary key definition
{
metadata.primary_key = KeyDescription::getKeyFromAST(args.storage_def->order_by->ptr(), metadata.columns, args.context);
/// and set it's definition_ast to nullptr (so isPrimaryKeyDefined()
/// will return false but hasPrimaryKey() will return true.
metadata.primary_key.definition_ast = nullptr;
}
if (args.storage_def->sample_by)
sample_by_ast = args.storage_def->sample_by->ptr();
metadata.sampling_key = KeyDescription::getKeyFromAST(args.storage_def->sample_by->ptr(), metadata.columns, args.context);
if (args.storage_def->ttl_table)
ttl_table_ast = args.storage_def->ttl_table->ptr();
metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(
args.storage_def->ttl_table->ptr(),
metadata.columns,
args.context,
metadata.primary_key);
if (args.query.columns_list && args.query.columns_list->indices)
for (auto & index : args.query.columns_list->indices->children)
indices_description.push_back(IndexDescription::getIndexFromAST(index, args.columns, args.context));
metadata.secondary_indices.push_back(IndexDescription::getIndexFromAST(index, args.columns, args.context));
if (args.query.columns_list && args.query.columns_list->constraints)
for (auto & constraint : args.query.columns_list->constraints->children)
metadata.constraints.constraints.push_back(constraint);
auto column_ttl_asts = args.columns.getColumnTTLs();
for (const auto & [name, ast] : column_ttl_asts)
{
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, args.columns, args.context, metadata.primary_key);
metadata.column_ttls_by_name[name] = new_ttl_entry;
}
storage_settings->loadFromQuery(*args.storage_def);
if (args.storage_def->settings)
settings_ast = args.storage_def->settings->ptr();
metadata.settings_changes = args.storage_def->settings->ptr();
}
else
{
@ -538,12 +574,23 @@ static StoragePtr create(const StorageFactory::Arguments & args)
/// If there is an expression for sampling
if (arg_cnt - arg_num == 3)
{
sample_by_ast = engine_args[arg_num];
metadata.sampling_key = KeyDescription::getKeyFromAST(engine_args[arg_num], metadata.columns, args.context);
++arg_num;
}
/// Now only two parameters remain - primary_key, index_granularity.
order_by_ast = engine_args[arg_num];
/// Get sorting key from engine arguments.
///
/// NOTE: store merging_param_key_arg as additional key column. We do it
/// before storage creation. After that storage will just copy this
/// column if sorting key will be changed.
metadata.sorting_key = KeyDescription::getSortingKeyFromAST(engine_args[arg_num], metadata.columns, args.context, merging_param_key_arg);
/// In old syntax primary_key always equals to sorting key.
metadata.primary_key = KeyDescription::getKeyFromAST(engine_args[arg_num], metadata.columns, args.context);
/// But it's not explicitely defined, so we evaluate definition to
/// nullptr
metadata.primary_key.definition_ast = nullptr;
++arg_num;
const auto * ast = engine_args[arg_num]->as<ASTLiteral>();
@ -559,18 +606,10 @@ static StoragePtr create(const StorageFactory::Arguments & args)
if (arg_num != arg_cnt)
throw Exception("Wrong number of engine arguments.", ErrorCodes::BAD_ARGUMENTS);
if (!args.attach && !indices_description.empty() && !args.local_context.getSettingsRef().allow_experimental_data_skipping_indices)
if (!args.attach && !metadata.secondary_indices.empty() && !args.local_context.getSettingsRef().allow_experimental_data_skipping_indices)
throw Exception("You must set the setting `allow_experimental_data_skipping_indices` to 1 " \
"before using data skipping indices.", ErrorCodes::BAD_ARGUMENTS);
StorageInMemoryMetadata metadata(args.columns, indices_description, args.constraints);
metadata.partition_by_ast = partition_by_ast;
metadata.order_by_ast = order_by_ast;
metadata.primary_key_ast = primary_key_ast;
metadata.ttl_for_table_ast = ttl_table_ast;
metadata.sample_by_ast = sample_by_ast;
metadata.settings_ast = settings_ast;
if (replicated)
return StorageReplicatedMergeTree::create(
zookeeper_path, replica_name, args.attach, args.table_id, args.relative_data_path,

120
src/Storages/SelectQueryDescription.cpp Normal file
View File

@ -0,0 +1,120 @@
#include <Storages/SelectQueryDescription.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTSelectQuery.h>
#include <Interpreters/getTableExpressions.h>
#include <Interpreters/AddDefaultDatabaseVisitor.h>
#include <Interpreters/Context.h>
namespace DB
{
namespace ErrorCodes
{
extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW;
extern const int LOGICAL_ERROR;
}
SelectQueryDescription::SelectQueryDescription(const SelectQueryDescription & other)
: select_table_id(other.select_table_id)
, select_query(other.select_query ? other.select_query->clone() : nullptr)
, inner_query(other.inner_query ? other.inner_query->clone() : nullptr)
{
}
SelectQueryDescription & SelectQueryDescription::SelectQueryDescription::operator=(const SelectQueryDescription & other)
{
if (&other == this)
return *this;
select_table_id = other.select_table_id;
if (other.select_query)
select_query = other.select_query->clone();
else
select_query.reset();
if (other.inner_query)
inner_query = other.inner_query->clone();
else
inner_query.reset();
return *this;
}
namespace
{
StorageID extractDependentTableFromSelectQuery(ASTSelectQuery & query, const Context & context, bool add_default_db = true)
{
if (add_default_db)
{
AddDefaultDatabaseVisitor visitor(context.getCurrentDatabase(), nullptr);
visitor.visit(query);
}
if (auto db_and_table = getDatabaseAndTable(query, 0))
{
return StorageID(db_and_table->database, db_and_table->table/*, db_and_table->uuid*/);
}
else if (auto subquery = extractTableExpression(query, 0))
{
auto * ast_select = subquery->as<ASTSelectWithUnionQuery>();
if (!ast_select)
throw Exception("Logical error while creating StorageMaterializedView. "
"Could not retrieve table name from select query.",
DB::ErrorCodes::LOGICAL_ERROR);
if (ast_select->list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for MATERIALIZED VIEW",
ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
auto & inner_query = ast_select->list_of_selects->children.at(0);
return extractDependentTableFromSelectQuery(inner_query->as<ASTSelectQuery &>(), context, false);
}
else
return StorageID::createEmpty();
}
void checkAllowedQueries(const ASTSelectQuery & query)
{
if (query.prewhere() || query.final() || query.sampleSize())
throw Exception("MATERIALIZED VIEW cannot have PREWHERE, SAMPLE or FINAL.", DB::ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
ASTPtr subquery = extractTableExpression(query, 0);
if (!subquery)
return;
if (const auto * ast_select = subquery->as<ASTSelectWithUnionQuery>())
{
if (ast_select->list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
const auto & inner_query = ast_select->list_of_selects->children.at(0);
checkAllowedQueries(inner_query->as<ASTSelectQuery &>());
}
}
}
SelectQueryDescription SelectQueryDescription::getSelectQueryFromASTForMatView(const ASTPtr & select, const Context & context)
{
auto & new_select = select->as<ASTSelectWithUnionQuery &>();
if (new_select.list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
SelectQueryDescription result;
result.inner_query = new_select.list_of_selects->children.at(0)->clone();
auto & select_query = result.inner_query->as<ASTSelectQuery &>();
checkAllowedQueries(select_query);
result.select_table_id = extractDependentTableFromSelectQuery(select_query, context);
result.select_query = select->clone();
return result;
}
}

27
src/Storages/SelectQueryDescription.h Normal file
View File

@ -0,0 +1,27 @@
#pragma once
#include <Interpreters/StorageID.h>
namespace DB
{
/// Select query for different view in storages
struct SelectQueryDescription
{
/// Table id for select query
StorageID select_table_id = StorageID::createEmpty();
/// Select query itself (ASTSelectWithUnionQuery)
ASTPtr select_query;
/// First query from select_query list
ASTPtr inner_query;
/// Parse description from select query for materialized view. Also
/// validates query.
static SelectQueryDescription getSelectQueryFromASTForMatView(const ASTPtr & select, const Context & context);
SelectQueryDescription() = default;
SelectQueryDescription(const SelectQueryDescription & other);
SelectQueryDescription & operator=(const SelectQueryDescription & other);
};
}

10
src/Storages/StorageBuffer.cpp
View File

@ -723,7 +723,7 @@ void StorageBuffer::reschedule()
flush_handle->scheduleAfter(std::min(min, max) * 1000);
}
void StorageBuffer::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */)
void StorageBuffer::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const
{
for (const auto & command : commands)
{
@ -775,10 +775,10 @@ void StorageBuffer::alter(const AlterCommands & params, const Context & context,
/// So that no blocks of the old structure remain.
optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context);
StorageInMemoryMetadata metadata = getInMemoryMetadata();
params.apply(metadata, context);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata);
setColumns(std::move(metadata.columns));
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
params.apply(new_metadata, context);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata);
setColumns(std::move(new_metadata.columns));
}

2
src/Storages/StorageBuffer.h
View File

@ -85,7 +85,7 @@ public:
bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context) const override;
void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override;
void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const override;
/// The structure of the subordinate table is not checked and does not change.
void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override;

10
src/Storages/StorageDistributed.cpp
View File

@ -538,7 +538,7 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const Context & c
}
void StorageDistributed::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */)
void StorageDistributed::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const
{
for (const auto & command : commands)
{
@ -559,10 +559,10 @@ void StorageDistributed::alter(const AlterCommands & params, const Context & con
auto table_id = getStorageID();
checkAlterIsPossible(params, context.getSettingsRef());
StorageInMemoryMetadata metadata = getInMemoryMetadata();
params.apply(metadata, context);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata);
setColumns(std::move(metadata.columns));
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
params.apply(new_metadata, context);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata);
setColumns(std::move(new_metadata.columns));
}

2
src/Storages/StorageDistributed.h
View File

@ -84,7 +84,7 @@ public:
void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override;
void renameOnDisk(const String & new_path_to_table_data);
void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override;
void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const override;
/// in the sub-tables, you need to manually add and delete columns
/// the structure of the sub-table is not checked

102
src/Storages/StorageInMemoryMetadata.cpp
View File

@ -1,13 +1,5 @@
#include <Storages/StorageInMemoryMetadata.h>
#include <Functions/IFunction.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <Interpreters/SyntaxAnalyzer.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTFunction.h>
#include <Storages/extractKeyExpressionList.h>
namespace DB
{
@ -25,94 +17,38 @@ StorageInMemoryMetadata::StorageInMemoryMetadata(const StorageInMemoryMetadata &
: columns(other.columns)
, secondary_indices(other.secondary_indices)
, constraints(other.constraints)
, partition_key(other.partition_key)
, primary_key(other.primary_key)
, sorting_key(other.sorting_key)
, sampling_key(other.sampling_key)
, column_ttls_by_name(other.column_ttls_by_name)
, table_ttl(other.table_ttl)
, settings_changes(other.settings_changes ? other.settings_changes->clone() : nullptr)
, select(other.select)
{
if (other.partition_by_ast)
partition_by_ast = other.partition_by_ast->clone();
if (other.order_by_ast)
order_by_ast = other.order_by_ast->clone();
if (other.primary_key_ast)
primary_key_ast = other.primary_key_ast->clone();
if (other.ttl_for_table_ast)
ttl_for_table_ast = other.ttl_for_table_ast->clone();
if (other.sample_by_ast)
sample_by_ast = other.sample_by_ast->clone();
if (other.settings_ast)
settings_ast = other.settings_ast->clone();
if (other.select)
select = other.select->clone();
}
StorageInMemoryMetadata & StorageInMemoryMetadata::operator=(const StorageInMemoryMetadata & other)
{
if (this == &other)
if (&other == this)
return *this;
columns = other.columns;
secondary_indices = other.secondary_indices;
constraints = other.constraints;
if (other.partition_by_ast)
partition_by_ast = other.partition_by_ast->clone();
partition_key = other.partition_key;
primary_key = other.primary_key;
sorting_key = other.sorting_key;
sampling_key = other.sampling_key;
column_ttls_by_name = other.column_ttls_by_name;
table_ttl = other.table_ttl;
if (other.settings_changes)
settings_changes = other.settings_changes->clone();
else
partition_by_ast.reset();
if (other.order_by_ast)
order_by_ast = other.order_by_ast->clone();
else
order_by_ast.reset();
if (other.primary_key_ast)
primary_key_ast = other.primary_key_ast->clone();
else
primary_key_ast.reset();
if (other.ttl_for_table_ast)
ttl_for_table_ast = other.ttl_for_table_ast->clone();
else
ttl_for_table_ast.reset();
if (other.sample_by_ast)
sample_by_ast = other.sample_by_ast->clone();
else
sample_by_ast.reset();
if (other.settings_ast)
settings_ast = other.settings_ast->clone();
else
settings_ast.reset();
if (other.select)
select = other.select->clone();
else
select.reset();
settings_changes.reset();
select = other.select;
return *this;
}
StorageMetadataKeyField StorageMetadataKeyField::getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context)
{
StorageMetadataKeyField result;
result.definition_ast = definition_ast;
result.expression_list_ast = extractKeyExpressionList(definition_ast);
if (result.expression_list_ast->children.empty())
return result;
const auto & children = result.expression_list_ast->children;
for (const auto & child : children)
result.column_names.emplace_back(child->getColumnName());
{
auto expr = result.expression_list_ast->clone();
auto syntax_result = SyntaxAnalyzer(context).analyze(expr, columns.getAllPhysical());
result.expression = ExpressionAnalyzer(expr, syntax_result, context).getActions(true);
result.sample_block = result.expression->getSampleBlock();
}
for (size_t i = 0; i < result.sample_block.columns(); ++i)
result.data_types.emplace_back(result.sample_block.getByPosition(i).type);
return result;
}
}

64
src/Storages/StorageInMemoryMetadata.h
View File

@ -1,10 +1,13 @@
#pragma once
#include <Storages/ColumnsDescription.h>
#include <Storages/IndicesDescription.h>
#include <Storages/ConstraintsDescription.h>
#include <Core/Field.h>
#include <Parsers/IAST_fwd.h>
#include <Storages/ColumnsDescription.h>
#include <Storages/ConstraintsDescription.h>
#include <Storages/IndicesDescription.h>
#include <Storages/KeyDescription.h>
#include <Storages/TTLDescription.h>
#include <Storages/SelectQueryDescription.h>
namespace DB
{
@ -22,55 +25,28 @@ struct StorageInMemoryMetadata
/// Table constraints. Currently supported for MergeTree only.
ConstraintsDescription constraints;
/// PARTITION BY expression. Currently supported for MergeTree only.
ASTPtr partition_by_ast = nullptr;
KeyDescription partition_key;
/// PRIMARY KEY expression. If absent, than equal to order_by_ast.
KeyDescription primary_key;
/// ORDER BY expression. Required field for all MergeTree tables
/// even in old syntax MergeTree(partition_key, order_by, ...)
ASTPtr order_by_ast = nullptr;
/// PRIMARY KEY expression. If absent, than equal to order_by_ast.
ASTPtr primary_key_ast = nullptr;
/// TTL expression for whole table. Supported for MergeTree only.
ASTPtr ttl_for_table_ast = nullptr;
KeyDescription sorting_key;
/// SAMPLE BY expression. Supported for MergeTree only.
ASTPtr sample_by_ast = nullptr;
KeyDescription sampling_key;
/// Separate ttl expressions for columns
TTLColumnsDescription column_ttls_by_name;
/// TTL expressions for table (Move and Rows)
TTLTableDescription table_ttl;
/// SETTINGS expression. Supported for MergeTree, Buffer and Kafka.
ASTPtr settings_ast = nullptr;
/// SELECT QUERY. Supported for MaterializedView only.
ASTPtr select = nullptr;
ASTPtr settings_changes;
/// SELECT QUERY. Supported for MaterializedView and View (have to support LiveView).
SelectQueryDescription select;
StorageInMemoryMetadata(const StorageInMemoryMetadata & other);
StorageInMemoryMetadata() = default;
StorageInMemoryMetadata(const ColumnsDescription & columns_, const IndicesDescription & secondary_indices_, const ConstraintsDescription & constraints_);
StorageInMemoryMetadata(const StorageInMemoryMetadata & other);
StorageInMemoryMetadata & operator=(const StorageInMemoryMetadata & other);
};
/// Common structure for primary, partition and other storage keys
struct StorageMetadataKeyField
{
/// User defined AST in CREATE/ALTER query. This field may be empty, but key
/// can exists because some of them maybe set implicitly (for example,
/// primary key in merge tree can be part of sorting key)
ASTPtr definition_ast;
/// ASTExpressionList with key fields, example: (x, toStartOfMonth(date))).
ASTPtr expression_list_ast;
/// Expression from expression_list_ast created by ExpressionAnalyzer. Useful,
/// when you need to get required columns for key, example: a, date, b.
ExpressionActionsPtr expression;
/// Sample block with key columns (names, types, empty column)
Block sample_block;
/// Column names in key definition, example: x, toStartOfMonth(date), a * b.
Names column_names;
/// Types from sample block ordered in columns order.
DataTypes data_types;
/// Parse key structure from key definition. Requires all columns, available
/// in storage.
static StorageMetadataKeyField getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context);
};
}

114
src/Storages/StorageMaterializedView.cpp
View File

@ -18,6 +18,7 @@
#include <Storages/AlterCommands.h>
#include <Storages/StorageFactory.h>
#include <Storages/ReadInOrderOptimizer.h>
#include <Storages/SelectQueryDescription.h>
#include <Common/typeid_cast.h>
#include <Processors/Sources/SourceFromInputStream.h>
@ -29,7 +30,6 @@ namespace DB
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
extern const int LOGICAL_ERROR;
extern const int INCORRECT_QUERY;
extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW;
}
@ -41,58 +41,6 @@ static inline String generateInnerTableName(const StorageID & view_id)
return ".inner." + view_id.getTableName();
}
static StorageID extractDependentTableFromSelectQuery(ASTSelectQuery & query, const Context & context, bool add_default_db = true)
{
if (add_default_db)
{
AddDefaultDatabaseVisitor visitor(context.getCurrentDatabase(), nullptr);
visitor.visit(query);
}
if (auto db_and_table = getDatabaseAndTable(query, 0))
{
return StorageID(db_and_table->database, db_and_table->table/*, db_and_table->uuid*/);
}
else if (auto subquery = extractTableExpression(query, 0))
{
auto * ast_select = subquery->as<ASTSelectWithUnionQuery>();
if (!ast_select)
throw Exception("Logical error while creating StorageMaterializedView. "
"Could not retrieve table name from select query.",
DB::ErrorCodes::LOGICAL_ERROR);
if (ast_select->list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for MATERIALIZED VIEW",
ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
auto & inner_query = ast_select->list_of_selects->children.at(0);
return extractDependentTableFromSelectQuery(inner_query->as<ASTSelectQuery &>(), context, false);
}
else
return StorageID::createEmpty();
}
static void checkAllowedQueries(const ASTSelectQuery & query)
{
if (query.prewhere() || query.final() || query.sampleSize())
throw Exception("MATERIALIZED VIEW cannot have PREWHERE, SAMPLE or FINAL.", DB::ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
ASTPtr subquery = extractTableExpression(query, 0);
if (!subquery)
return;
if (const auto * ast_select = subquery->as<ASTSelectWithUnionQuery>())
{
if (ast_select->list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
const auto & inner_query = ast_select->list_of_selects->children.at(0);
checkAllowedQueries(inner_query->as<ASTSelectQuery &>());
}
}
StorageMaterializedView::StorageMaterializedView(
const StorageID & table_id_,
@ -117,13 +65,8 @@ StorageMaterializedView::StorageMaterializedView(
if (query.select->list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
select = query.select->clone();
inner_query = query.select->list_of_selects->children.at(0);
auto & select_query = inner_query->as<ASTSelectQuery &>();
checkAllowedQueries(select_query);
select_table_id = extractDependentTableFromSelectQuery(select_query, local_context);
auto select = SelectQueryDescription::getSelectQueryFromASTForMatView(query.select->clone(), local_context);
setSelectQuery(select);
if (!has_inner_table)
target_table_id = query.to_table_id;
@ -152,15 +95,8 @@ StorageMaterializedView::StorageMaterializedView(
target_table_id = DatabaseCatalog::instance().getTable({manual_create_query->database, manual_create_query->table}, global_context)->getStorageID();
}
if (!select_table_id.empty())
DatabaseCatalog::instance().addDependency(select_table_id, getStorageID());
}
StorageInMemoryMetadata StorageMaterializedView::getInMemoryMetadata() const
{
StorageInMemoryMetadata result(getColumns(), getSecondaryIndices(), getConstraints());
result.select = getSelectQuery();
return result;
if (!select.select_table_id.empty())
DatabaseCatalog::instance().addDependency(select.select_table_id, getStorageID());
}
QueryProcessingStage::Enum StorageMaterializedView::getQueryProcessingStage(const Context & context, QueryProcessingStage::Enum to_stage, const ASTPtr & query_ptr) const
@ -222,8 +158,9 @@ static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context,
void StorageMaterializedView::drop()
{
auto table_id = getStorageID();
if (!select_table_id.empty())
DatabaseCatalog::instance().removeDependency(select_table_id, table_id);
const auto & select_query = getSelectQuery();
if (!select_query.select_table_id.empty())
DatabaseCatalog::instance().removeDependency(select_query.select_table_id, table_id);
if (has_inner_table && tryGetTargetTable())
executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_table_id);
@ -256,36 +193,27 @@ void StorageMaterializedView::alter(
{
lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout);
auto table_id = getStorageID();
StorageInMemoryMetadata metadata = getInMemoryMetadata();
params.apply(metadata, context);
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
params.apply(new_metadata, context);
/// start modify query
if (context.getSettingsRef().allow_experimental_alter_materialized_view_structure)
{
auto & new_select = metadata.select->as<ASTSelectWithUnionQuery &>();
const auto & new_select = new_metadata.select;
const auto & old_select = getSelectQuery();
if (new_select.list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
DatabaseCatalog::instance().updateDependency(old_select.select_table_id, table_id, new_select.select_table_id, table_id);
auto & new_inner_query = new_select.list_of_selects->children.at(0);
auto & select_query = new_inner_query->as<ASTSelectQuery &>();
checkAllowedQueries(select_query);
auto new_select_table_id = extractDependentTableFromSelectQuery(select_query, context);
DatabaseCatalog::instance().updateDependency(select_table_id, table_id, new_select_table_id, table_id);
select_table_id = new_select_table_id;
select = metadata.select;
inner_query = new_inner_query;
setSelectQuery(new_select);
}
/// end modify query
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata);
setColumns(std::move(metadata.columns));
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata);
setColumns(std::move(new_metadata.columns));
}
void StorageMaterializedView::checkAlterIsPossible(const AlterCommands & commands, const Settings & settings)
void StorageMaterializedView::checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const
{
if (settings.allow_experimental_alter_materialized_view_structure)
{
@ -349,15 +277,17 @@ void StorageMaterializedView::renameInMemory(const StorageID & new_table_id)
}
IStorage::renameInMemory(new_table_id);
const auto & select_query = getSelectQuery();
// TODO Actually we don't need to update dependency if MV has UUID, but then db and table name will be outdated
DatabaseCatalog::instance().updateDependency(select_table_id, old_table_id, select_table_id, getStorageID());
DatabaseCatalog::instance().updateDependency(select_query.select_table_id, old_table_id, select_query.select_table_id, getStorageID());
}
void StorageMaterializedView::shutdown()
{
const auto & select_query = getSelectQuery();
/// Make sure the dependency is removed after DETACH TABLE
if (!select_table_id.empty())
DatabaseCatalog::instance().removeDependency(select_table_id, getStorageID());
if (!select_query.select_table_id.empty())
DatabaseCatalog::instance().removeDependency(select_query.select_table_id, getStorageID());
}
StoragePtr StorageMaterializedView::getTargetTable() const

11
src/Storages/StorageMaterializedView.h
View File

@ -19,12 +19,8 @@ public:
std::string getName() const override { return "MaterializedView"; }
bool isView() const override { return true; }
ASTPtr getSelectQuery() const { return select->clone(); }
ASTPtr getInnerQuery() const { return inner_query->clone(); }
bool hasInnerTable() const { return has_inner_table; }
StorageInMemoryMetadata getInMemoryMetadata() const override;
bool supportsSampling() const override { return getTargetTable()->supportsSampling(); }
bool supportsPrewhere() const override { return getTargetTable()->supportsPrewhere(); }
bool supportsFinal() const override { return getTargetTable()->supportsFinal(); }
@ -45,7 +41,7 @@ public:
void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override;
void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) override;
void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const override;
void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override;
@ -76,14 +72,9 @@ public:
Strings getDataPaths() const override;
private:
/// Can be empty if SELECT query doesn't contain table
StorageID select_table_id = StorageID::createEmpty();
/// Will be initialized in constructor
StorageID target_table_id = StorageID::createEmpty();
ASTPtr select;
ASTPtr inner_query;
Context & global_context;
bool has_inner_table = false;

2
src/Storages/StorageMerge.cpp
View File

@ -372,7 +372,7 @@ DatabaseTablesIteratorPtr StorageMerge::getDatabaseIterator(const Context & cont
}
void StorageMerge::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */)
void StorageMerge::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const
{
for (const auto & command : commands)
{

2
src/Storages/StorageMerge.h
View File

@ -37,7 +37,7 @@ public:
size_t max_block_size,
unsigned num_streams) override;
void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override;
void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const override;
/// you need to add and remove columns in the sub-tables manually
/// the structure of sub-tables is not checked

24
src/Storages/StorageMergeTree.cpp
View File

@ -52,7 +52,7 @@ namespace ActionLocks
StorageMergeTree::StorageMergeTree(
const StorageID & table_id_,
const String & relative_data_path_,
const StorageInMemoryMetadata & metadata,
const StorageInMemoryMetadata & metadata_,
bool attach,
Context & context_,
const String & date_column_name,
@ -62,7 +62,7 @@ StorageMergeTree::StorageMergeTree(
: MergeTreeData(
table_id_,
relative_data_path_,
metadata,
metadata_,
context_,
date_column_name,
merging_params_,
@ -256,20 +256,20 @@ void StorageMergeTree::alter(
{
auto table_id = getStorageID();
StorageInMemoryMetadata metadata = getInMemoryMetadata();
auto maybe_mutation_commands = commands.getMutationCommands(metadata, context.getSettingsRef().materialize_ttl_after_modify, context);
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
auto maybe_mutation_commands = commands.getMutationCommands(new_metadata, context.getSettingsRef().materialize_ttl_after_modify, context);
String mutation_file_name;
Int64 mutation_version = -1;
commands.apply(metadata, context);
commands.apply(new_metadata, context);
/// This alter can be performed at metadata level only
/// This alter can be performed at new_metadata level only
if (commands.isSettingsAlter())
{
lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout);
changeSettings(metadata.settings_ast, table_lock_holder);
changeSettings(new_metadata.settings_changes, table_lock_holder);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata);
}
else
{
@ -278,13 +278,13 @@ void StorageMergeTree::alter(
auto merges_block = getActionLock(ActionLocks::PartsMerge);
lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout);
changeSettings(metadata.settings_ast, table_lock_holder);
changeSettings(new_metadata.settings_changes, table_lock_holder);
/// Reinitialize primary key because primary key column types might have changed.
setProperties(metadata);
setProperties(new_metadata);
setTTLExpressions(metadata.columns, metadata.ttl_for_table_ast);
setTTLExpressions(new_metadata);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata);
if (!maybe_mutation_commands.empty())
mutation_version = startMutation(maybe_mutation_commands, mutation_file_name);

10
src/Storages/StorageNull.cpp
View File

@ -32,7 +32,7 @@ void registerStorageNull(StorageFactory & factory)
});
}
void StorageNull::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */)
void StorageNull::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const
{
for (const auto & command : commands)
{
@ -51,10 +51,10 @@ void StorageNull::alter(
lockStructureExclusively(table_lock_holder, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout);
auto table_id = getStorageID();
StorageInMemoryMetadata metadata = getInMemoryMetadata();
params.apply(metadata, context);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, metadata);
setColumns(std::move(metadata.columns));
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
params.apply(new_metadata, context);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata);
setColumns(std::move(new_metadata.columns));
}
}

2
src/Storages/StorageNull.h
View File

@ -40,7 +40,7 @@ public:
return std::make_shared<NullBlockOutputStream>(getSampleBlock());
}
void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override;
void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) const override;
void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override;

75
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -160,7 +160,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
bool attach,
const StorageID & table_id_,
const String & relative_data_path_,
const StorageInMemoryMetadata & metadata,
const StorageInMemoryMetadata & metadata_,
Context & context_,
const String & date_column_name,
const MergingParams & merging_params_,
@ -168,7 +168,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
bool has_force_restore_data_flag)
: MergeTreeData(table_id_,
relative_data_path_,
metadata,
metadata_,
context_,
date_column_name,
merging_params_,
@ -483,7 +483,7 @@ bool StorageReplicatedMergeTree::createTableIfNotExists()
LOG_DEBUG(log, "Creating table {}", zookeeper_path);
/// We write metadata of table so that the replicas can check table parameters with them.
String metadata = ReplicatedMergeTreeTableMetadata(*this).toString();
String metadata_str = ReplicatedMergeTreeTableMetadata(*this).toString();
Coordination::Requests ops;
ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path, "", zkutil::CreateMode::Persistent));
@ -492,7 +492,7 @@ bool StorageReplicatedMergeTree::createTableIfNotExists()
ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/dropped", "", zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path + "/dropped", -1));
ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/metadata", metadata,
ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/metadata", metadata_str,
zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/columns", getColumns().toString(),
zkutil::CreateMode::Persistent));
@ -527,7 +527,7 @@ bool StorageReplicatedMergeTree::createTableIfNotExists()
zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/is_lost", "0",
zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata", metadata,
ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata", metadata_str,
zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/columns", getColumns().toString(),
zkutil::CreateMode::Persistent));
@ -751,9 +751,18 @@ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr
void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff)
{
StorageInMemoryMetadata metadata = getInMemoryMetadata();
if (new_columns != metadata.columns)
metadata.columns = new_columns;
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
if (new_columns != new_metadata.columns)
{
new_metadata.columns = new_columns;
new_metadata.column_ttls_by_name.clear();
for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs())
{
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, global_context, new_metadata.primary_key);
new_metadata.column_ttls_by_name[name] = new_ttl_entry;
}
}
if (!metadata_diff.empty())
{
@ -762,43 +771,51 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column
ParserNotEmptyExpressionList parser(false);
auto new_sorting_key_expr_list = parseQuery(parser, metadata_diff.new_sorting_key, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
ASTPtr order_by_ast;
if (new_sorting_key_expr_list->children.size() == 1)
metadata.order_by_ast = new_sorting_key_expr_list->children[0];
order_by_ast = new_sorting_key_expr_list->children[0];
else
{
auto tuple = makeASTFunction("tuple");
tuple->arguments->children = new_sorting_key_expr_list->children;
metadata.order_by_ast = tuple;
order_by_ast = tuple;
}
auto & sorting_key = new_metadata.sorting_key;
auto & primary_key = new_metadata.primary_key;
if (!isPrimaryKeyDefined())
sorting_key.recalculateWithNewAST(order_by_ast, new_metadata.columns, global_context);
if (primary_key.definition_ast == nullptr)
{
/// Primary and sorting key become independent after this ALTER so we have to
/// save the old ORDER BY expression as the new primary key.
metadata.primary_key_ast = getSortingKeyAST()->clone();
auto old_sorting_key_ast = getSortingKey().definition_ast;
primary_key = KeyDescription::getKeyFromAST(
old_sorting_key_ast, new_metadata.columns, global_context);
}
}
if (metadata_diff.skip_indices_changed)
metadata.secondary_indices = IndicesDescription::parse(metadata_diff.new_skip_indices, new_columns, global_context);
new_metadata.secondary_indices = IndicesDescription::parse(metadata_diff.new_skip_indices, new_columns, global_context);
if (metadata_diff.constraints_changed)
metadata.constraints = ConstraintsDescription::parse(metadata_diff.new_constraints);
new_metadata.constraints = ConstraintsDescription::parse(metadata_diff.new_constraints);
if (metadata_diff.ttl_table_changed)
{
ParserTTLExpressionList parser;
metadata.ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
auto ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl_for_table_ast, new_metadata.columns, global_context, new_metadata.primary_key);
}
}
auto table_id = getStorageID();
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(global_context, table_id, metadata);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(global_context, table_id, new_metadata);
/// Even if the primary/sorting keys didn't change we must reinitialize it
/// because primary key column types might have changed.
setProperties(metadata);
setTTLExpressions(new_columns, metadata.ttl_for_table_ast);
setProperties(new_metadata);
setTTLExpressions(new_metadata);
}
@ -3628,13 +3645,13 @@ void StorageReplicatedMergeTree::alter(
table_lock_holder, query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout);
/// We don't replicate storage_settings_ptr ALTER. It's local operation.
/// Also we don't upgrade alter lock to table structure lock.
StorageInMemoryMetadata metadata = getInMemoryMetadata();
params.apply(metadata, query_context);
StorageInMemoryMetadata future_metadata = getInMemoryMetadata();
params.apply(future_metadata, query_context);
changeSettings(metadata.settings_ast, table_lock_holder);
changeSettings(future_metadata.settings_changes, table_lock_holder);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id, metadata);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id, future_metadata);
return;
}
@ -3667,11 +3684,11 @@ void StorageReplicatedMergeTree::alter(
params.apply(future_metadata, query_context);
ReplicatedMergeTreeTableMetadata future_metadata_in_zk(*this);
if (ast_to_str(future_metadata.order_by_ast) != ast_to_str(current_metadata.order_by_ast))
future_metadata_in_zk.sorting_key = serializeAST(*extractKeyExpressionList(future_metadata.order_by_ast));
if (ast_to_str(future_metadata.sorting_key.definition_ast) != ast_to_str(current_metadata.sorting_key.definition_ast))
future_metadata_in_zk.sorting_key = serializeAST(*future_metadata.sorting_key.expression_list_ast);
if (ast_to_str(future_metadata.ttl_for_table_ast) != ast_to_str(current_metadata.ttl_for_table_ast))
future_metadata_in_zk.ttl_table = serializeAST(*future_metadata.ttl_for_table_ast);
if (ast_to_str(future_metadata.table_ttl.definition_ast) != ast_to_str(current_metadata.table_ttl.definition_ast))
future_metadata_in_zk.ttl_table = serializeAST(*future_metadata.table_ttl.definition_ast);
String new_indices_str = future_metadata.secondary_indices.toString();
if (new_indices_str != current_metadata.secondary_indices.toString())
@ -3689,13 +3706,13 @@ void StorageReplicatedMergeTree::alter(
String new_columns_str = future_metadata.columns.toString();
ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, -1));
if (ast_to_str(current_metadata.settings_ast) != ast_to_str(future_metadata.settings_ast))
if (ast_to_str(current_metadata.settings_changes) != ast_to_str(future_metadata.settings_changes))
{
lockStructureExclusively(
table_lock_holder, query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout);
/// Just change settings
current_metadata.settings_ast = future_metadata.settings_ast;
changeSettings(current_metadata.settings_ast, table_lock_holder);
current_metadata.settings_changes = future_metadata.settings_changes;
changeSettings(current_metadata.settings_changes, table_lock_holder);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id, current_metadata);
}

2
src/Storages/StorageReplicatedMergeTree.h
View File

@ -550,7 +550,7 @@ protected:
bool attach,
const StorageID & table_id_,
const String & relative_data_path_,
const StorageInMemoryMetadata & metadata,
const StorageInMemoryMetadata & metadata_,
Context & context_,
const String & date_column_name,
const MergingParams & merging_params_,

9
src/Storages/StorageView.cpp
View File

@ -10,6 +10,7 @@
#include <Storages/StorageView.h>
#include <Storages/StorageFactory.h>
#include <Storages/SelectQueryDescription.h>
#include <Common/typeid_cast.h>
@ -42,7 +43,10 @@ StorageView::StorageView(
if (!query.select)
throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY);
inner_query = query.select->ptr();
SelectQueryDescription description;
description.inner_query = query.select->ptr();
setSelectQuery(description);
}
@ -56,7 +60,8 @@ Pipes StorageView::read(
{
Pipes pipes;
ASTPtr current_inner_query = inner_query;
ASTPtr current_inner_query = getSelectQuery().inner_query;
if (query_info.view_query)
{
if (!query_info.view_query->as<ASTSelectWithUnionQuery>())

5
src/Storages/StorageView.h
View File

@ -31,15 +31,12 @@ public:
void replaceWithSubquery(ASTSelectQuery & select_query, ASTPtr & view_name) const
{
replaceWithSubquery(select_query, inner_query->clone(), view_name);
replaceWithSubquery(select_query, getSelectQuery().inner_query->clone(), view_name);
}
static void replaceWithSubquery(ASTSelectQuery & outer_query, ASTPtr view_query, ASTPtr & view_name);
static ASTPtr restoreViewName(ASTSelectQuery & select_query, const ASTPtr & view_name);
private:
ASTPtr inner_query;
protected:
StorageView(
const StorageID & table_id_,

112
src/Storages/TTLDescription.cpp
View File

@ -23,6 +23,29 @@ extern const int BAD_ARGUMENTS;
extern const int BAD_TTL_EXPRESSION;
}
TTLAggregateDescription::TTLAggregateDescription(const TTLAggregateDescription & other)
: column_name(other.column_name)
, expression_result_column_name(other.expression_result_column_name)
{
if (other.expression)
expression = std::make_shared<ExpressionActions>(*other.expression);
}
TTLAggregateDescription & TTLAggregateDescription::operator=(const TTLAggregateDescription & other)
{
if (&other == this)
return *this;
column_name = other.column_name;
expression_result_column_name = other.expression_result_column_name;
if (other.expression)
expression = std::make_shared<ExpressionActions>(*other.expression);
else
expression.reset();
return *this;
}
namespace
{
@ -55,11 +78,60 @@ void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const Strin
}
TTLDescription::TTLDescription(const TTLDescription & other)
: mode(other.mode)
, expression_ast(other.expression_ast ? other.expression_ast->clone() : nullptr)
, result_column(other.result_column)
, where_result_column(other.where_result_column)
, group_by_keys(other.group_by_keys)
, set_parts(other.set_parts)
, aggregate_descriptions(other.aggregate_descriptions)
, destination_type(other.destination_type)
, destination_name(other.destination_name)
{
if (other.expression)
expression = std::make_shared<ExpressionActions>(*other.expression);
if (other.where_expression)
where_expression = std::make_shared<ExpressionActions>(*other.where_expression);
}
TTLDescription & TTLDescription::operator=(const TTLDescription & other)
{
if (&other == this)
return *this;
mode = other.mode;
if (other.expression_ast)
expression_ast = other.expression_ast->clone();
else
expression_ast.reset();
if (other.expression)
expression = std::make_shared<ExpressionActions>(*other.expression);
else
expression.reset();
result_column = other.result_column;
if (other.where_expression)
where_expression = std::make_shared<ExpressionActions>(*other.where_expression);
else
where_expression.reset();
where_result_column = other.where_result_column;
group_by_keys = other.group_by_keys;
set_parts = other.set_parts;
aggregate_descriptions = other.aggregate_descriptions;
destination_type = other.destination_type;
destination_name = other.destination_name;
return * this;
}
TTLDescription TTLDescription::getTTLFromAST(
const ASTPtr & definition_ast,
const ColumnsDescription & columns,
const Context & context,
const StorageMetadataKeyField & primary_key)
const KeyDescription & primary_key)
{
TTLDescription result;
const auto * ttl_element = definition_ast->as<ASTTTLElement>();
@ -180,8 +252,12 @@ TTLDescription TTLDescription::getTTLFromAST(
auto syntax_result = SyntaxAnalyzer(context).analyze(value, columns.getAllPhysical(), {}, true);
auto expr_analyzer = ExpressionAnalyzer(value, syntax_result, context);
result.set_parts.emplace_back(TTLAggregateDescription{
name, value->getColumnName(), expr_analyzer.getActions(false)});
TTLAggregateDescription set_part;
set_part.column_name = name;
set_part.expression_result_column_name = value->getColumnName();
set_part.expression = expr_analyzer.getActions(false);
result.set_parts.emplace_back(set_part);
for (const auto & descr : expr_analyzer.getAnalyzedData().aggregate_descriptions)
result.aggregate_descriptions.push_back(descr);
@ -195,4 +271,34 @@ TTLDescription TTLDescription::getTTLFromAST(
return result;
}
TTLTableDescription TTLTableDescription::getTTLForTableFromAST(
const ASTPtr & definition_ast,
const ColumnsDescription & columns,
const Context & context,
const KeyDescription & primary_key)
{
TTLTableDescription result;
if (!definition_ast)
return result;
result.definition_ast = definition_ast->clone();
bool seen_delete_ttl = false;
for (const auto & ttl_element_ptr : definition_ast->children)
{
auto ttl = TTLDescription::getTTLFromAST(ttl_element_ptr, columns, context, primary_key);
if (ttl.destination_type == DataDestinationType::DELETE)
{
if (seen_delete_ttl)
throw Exception("More than one DELETE TTL expression is not allowed", ErrorCodes::BAD_TTL_EXPRESSION);
result.rows_ttl = ttl;
seen_delete_ttl = true;
}
else
result.move_ttl.emplace_back(std::move(ttl));
}
return result;
}
}

15
src/Storages/TTLDescription.h
View File

@ -2,9 +2,9 @@
#include <Parsers/IAST_fwd.h>
#include <Storages/DataDestinationType.h>
#include <Storages/ColumnsDescription.h>
#include <Storages/KeyDescription.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/AggregateDescription.h>
#include <Storages/StorageInMemoryMetadata.h>
#include <Storages/TTLMode.h>
namespace DB
@ -25,6 +25,10 @@ struct TTLAggregateDescription
/// Expressions to calculate the value of assignment expression
ExpressionActionsPtr expression;
TTLAggregateDescription() = default;
TTLAggregateDescription(const TTLAggregateDescription & other);
TTLAggregateDescription & operator=(const TTLAggregateDescription & other);
};
using TTLAggregateDescriptions = std::vector<TTLAggregateDescription>;
@ -73,7 +77,11 @@ struct TTLDescription
/// Parse TTL structure from definition. Able to parse both column and table
/// TTLs.
static TTLDescription getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, const StorageMetadataKeyField & primary_key);
static TTLDescription getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, const KeyDescription & primary_key);
TTLDescription() = default;
TTLDescription(const TTLDescription & other);
TTLDescription & operator=(const TTLDescription & other);
};
/// Mapping from column name to column TTL
@ -93,6 +101,9 @@ struct TTLTableDescription
/// Moving data TTL (to other disks or volumes)
TTLDescriptions move_ttl;
static TTLTableDescription getTTLForTableFromAST(
const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, const KeyDescription & primary_key);
};
}

2
src/Storages/ya.make
View File

@ -177,6 +177,8 @@ SRCS(
VirtualColumnUtils.cpp
extractKeyExpressionList.cpp
TTLDescription.cpp
KeyDescription.cpp
SelectQueryDescription.cpp
)
END()

3
tests/queries/0_stateless/00933_alter_ttl.sql
View File

@ -21,6 +21,7 @@ drop table if exists ttl;
create table ttl (d Date, a Int) engine = MergeTree order by tuple() partition by toDayOfMonth(d);
alter table ttl modify column a Int ttl d + interval 1 day;
desc table ttl;
alter table ttl modify column d Int ttl d + interval 1 day; -- { serverError 524 }
alter table ttl modify column d Int ttl d + interval 1 day; -- { serverError 43 }
alter table ttl modify column d DateTime ttl d + interval 1 day; -- { serverError 524 }
drop table if exists ttl;

12
tests/queries/0_stateless/01213_alter_rename_primary_key_zookeeper.sql
View File

@ -17,11 +17,11 @@ INSERT INTO table_for_rename_pk SELECT toDate('2019-10-01') + number % 3, number
SELECT key1, value1 FROM table_for_rename_pk WHERE key1 = 1 AND key2 = 1 AND key3 = 1;
ALTER TABLE table_for_rename_pk RENAME COLUMN key1 TO renamed_key1; --{serverError 524}
ALTER TABLE table_for_rename_pk RENAME COLUMN key1 TO renamed_key1; --{serverError 47}
ALTER TABLE table_for_rename_pk RENAME COLUMN key3 TO renamed_key3; --{serverError 524}
ALTER TABLE table_for_rename_pk RENAME COLUMN key3 TO renamed_key3; --{serverError 47}
ALTER TABLE table_for_rename_pk RENAME COLUMN key2 TO renamed_key2; --{serverError 524}
ALTER TABLE table_for_rename_pk RENAME COLUMN key2 TO renamed_key2; --{serverError 47}
DROP TABLE IF EXISTS table_for_rename_pk NO DELAY;
SELECT sleep(1) FORMAT Null;
@ -45,11 +45,11 @@ PRIMARY KEY (key1, key2);
INSERT INTO table_for_rename_with_primary_key SELECT toDate('2019-10-01') + number % 3, number, number, number, toString(number), toString(number) from numbers(9);
ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key1 TO renamed_key1; --{serverError 524}
ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key1 TO renamed_key1; --{serverError 47}
ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key2 TO renamed_key2; --{serverError 524}
ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key2 TO renamed_key2; --{serverError 47}
ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key3 TO renamed_key3; --{serverError 524}
ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN key3 TO renamed_key3; --{serverError 47}
ALTER TABLE table_for_rename_with_primary_key RENAME COLUMN value1 TO renamed_value1; --{serverError 524}

6
tests/queries/0_stateless/01277_alter_rename_column_constraint.sql
View File

@ -15,7 +15,7 @@ PARTITION BY date
ORDER BY key;
INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(9);
INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(9); ; --{serverError 469}
INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(9); --{serverError 469}
SELECT * FROM table_for_rename ORDER BY key;
@ -26,7 +26,7 @@ SELECT * FROM table_for_rename ORDER BY key;
SELECT '-- insert after rename --';
INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(10, 10);
INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(10, 10); ; --{serverError 469}
INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number) from numbers(10, 10); --{serverError 469}
SELECT * FROM table_for_rename ORDER BY key;
SELECT '-- rename columns back --';
@ -37,7 +37,7 @@ SELECT * FROM table_for_rename ORDER BY key;
SELECT '-- insert after rename column --';
INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1), toString(number + 2) from numbers(20,10);
INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number + 2) from numbers(20, 10); ; --{serverError 469}
INSERT INTO table_for_rename SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number + 2) from numbers(20, 10); --{serverError 469}
SELECT * FROM table_for_rename ORDER BY key;
DROP TABLE IF EXISTS table_for_rename;