thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 17:41:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

More usage of metadata

Browse Source
This commit is contained in:
alesapin 2019-12-27 19:34:50 +03:00
parent fcf3552750
commit 2d40b33120
9 changed files with 62 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
dbms/src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -114,17 +114,9 @@ MergeTreeData::MergeTreeData(
const String & database_,
const String & table_,
const String & relative_data_path_,
const ColumnsDescription & columns_,
const IndicesDescription & indices_,
const ConstraintsDescription & constraints_,
const StorageInMemoryMetadata & metadata,
Context & context_,
const String & date_column_name,
const ASTPtr & partition_by_ast_,
const ASTPtr & order_by_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sample_by_ast_,
const ASTPtr & ttl_table_ast_,
const ASTPtr & settings_ast_,
const MergingParams & merging_params_,
std::unique_ptr<MergeTreeSettings> storage_settings_,
bool require_part_metadata_,
@ -132,9 +124,6 @@ MergeTreeData::MergeTreeData(
BrokenPartCallback broken_part_callback_)
: global_context(context_)
, merging_params(merging_params_)
, partition_by_ast(partition_by_ast_)
, sample_by_ast(sample_by_ast_)
, settings_ast(settings_ast_)
, require_part_metadata(require_part_metadata_)
, database_name(database_)
, table_name(table_)
@ -149,7 +138,7 @@ MergeTreeData::MergeTreeData(
, parts_mover(this)
{
const auto settings = getSettings();
setProperties(order_by_ast_, primary_key_ast_, columns_, indices_, constraints_);
setProperties(metadata);
/// NOTE: using the same columns list as is read when performing actual merges.
merging_params.check(getColumns().getAllPhysical());
@ -191,7 +180,7 @@ MergeTreeData::MergeTreeData(
min_format_version = MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING;
}
setTTLExpressions(columns_.getColumnTTLs(), ttl_table_ast_);
setTTLExpressions(metadata.columns.getColumnTTLs(), metadata.ttl_for_table_ast);
// format_file always contained on any data path
String version_file_path;
@ -304,17 +293,14 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam
}
void MergeTreeData::setProperties(
const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast,
const ColumnsDescription & new_columns, const IndicesDescription & indices_description,
const ConstraintsDescription & constraints_description, bool only_check)
void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool only_check)
{
if (!new_order_by_ast)
if (!metadata.order_by_ast)
throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS);
ASTPtr new_sorting_key_expr_list = extractKeyExpressionList(new_order_by_ast);
ASTPtr new_primary_key_expr_list = new_primary_key_ast
? extractKeyExpressionList(new_primary_key_ast) : new_sorting_key_expr_list->clone();
ASTPtr new_sorting_key_expr_list = extractKeyExpressionList(metadata.order_by_ast);
ASTPtr new_primary_key_expr_list = metadata.primary_key_ast
? extractKeyExpressionList(metadata.primary_key_ast) : new_sorting_key_expr_list->clone();
if (merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing)
new_sorting_key_expr_list->children.push_back(std::make_shared<ASTIdentifier>(merging_params.version_column));
@ -346,7 +332,7 @@ void MergeTreeData::setProperties(
}
}
auto all_columns = new_columns.getAllPhysical();
auto all_columns = metadata.columns.getAllPhysical();
/// Order by check AST
if (order_by_ast && only_check)
@ -384,7 +370,7 @@ void MergeTreeData::setProperties(
"added to the sorting key. You can add expressions that use only the newly added columns",
ErrorCodes::BAD_ARGUMENTS);
if (new_columns.getDefaults().count(col))
if (metadata.columns.getDefaults().count(col))
throw Exception("Newly added column " + col + " has a default expression, so adding "
"expressions that use it to the sorting key is forbidden",
ErrorCodes::BAD_ARGUMENTS);
@ -419,11 +405,11 @@ void MergeTreeData::setProperties(
MergeTreeIndices new_indices;
if (!indices_description.indices.empty())
if (!metadata.indices.indices.empty())
{
std::set<String> indices_names;
for (const auto & index_ast : indices_description.indices)
for (const auto & index_ast : metadata.indices.indices)
{
const auto & index_decl = std::dynamic_pointer_cast<ASTIndexDeclaration>(index_ast);
@ -460,24 +446,24 @@ void MergeTreeData::setProperties(
if (!only_check)
{
setColumns(std::move(new_columns));
setColumns(std::move(metadata.columns));
order_by_ast = new_order_by_ast;
order_by_ast = metadata.order_by_ast;
sorting_key_columns = std::move(new_sorting_key_columns);
sorting_key_expr_ast = std::move(new_sorting_key_expr_list);
sorting_key_expr = std::move(new_sorting_key_expr);
primary_key_ast = new_primary_key_ast;
primary_key_ast = metadata.primary_key_ast;
primary_key_columns = std::move(new_primary_key_columns);
primary_key_expr_ast = std::move(new_primary_key_expr_list);
primary_key_expr = std::move(new_primary_key_expr);
primary_key_sample = std::move(new_primary_key_sample);
primary_key_data_types = std::move(new_primary_key_data_types);
setIndices(indices_description);
setIndices(metadata.indices);
skip_indices = std::move(new_indices);
setConstraints(constraints_description);
setConstraints(metadata.constraints);
primary_key_and_skip_indices_expr = new_indices_with_primary_key_expr;
sorting_key_and_skip_indices_expr = new_indices_with_sorting_key_expr;

15
dbms/src/Storages/MergeTree/MergeTreeData.h
View File

@ -332,17 +332,9 @@ public:
/// attach - whether the existing table is attached or the new table is created.
MergeTreeData(const String & database_, const String & table_,
const String & relative_data_path_,
const ColumnsDescription & columns_,
const IndicesDescription & indices_,
const ConstraintsDescription & constraints_,
const StorageInMemoryMetadata & metadata,
Context & context_,
const String & date_column_name,
const ASTPtr & partition_by_ast_,
const ASTPtr & order_by_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported.
const ASTPtr & ttl_table_ast_,
const ASTPtr & settings_ast_,
const MergingParams & merging_params_,
std::unique_ptr<MergeTreeSettings> settings_,
bool require_part_metadata_,
@ -900,10 +892,7 @@ protected:
std::mutex clear_old_temporary_directories_mutex;
/// Mutex for settings usage
void setProperties(const ASTPtr & new_order_by_ast, const ASTPtr & new_primary_key_ast,
const ColumnsDescription & new_columns,
const IndicesDescription & indices_description,
const ConstraintsDescription & constraints_description, bool only_check = false);
void setProperties(const StorageInMemoryMetadata & metadata, bool only_check = false);
void initPartitionKey();

2
dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp
View File

@ -36,7 +36,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr
/// This code may looks strange, but previously we had only one entity: PRIMARY KEY (or ORDER BY, it doesn't matter)
/// Now we have two different entities ORDER BY and it's optional prefix -- PRIMARY KEY.
/// In most cases users doesn't specify PRIMARY KEY and semantically it's equal to ORDER BY.
/// In most cases user doesn't specify PRIMARY KEY and semantically it's equal to ORDER BY.
/// So rules in zookeeper metadata is following:
/// - When we have only ORDER BY, than store it in "primary key:" row of /metadata
/// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata

21
dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp
View File

@ -643,18 +643,27 @@ static StoragePtr create(const StorageFactory::Arguments & args)
throw Exception("You must set the setting `allow_experimental_data_skipping_indices` to 1 " \
"before using data skipping indices.", ErrorCodes::BAD_ARGUMENTS);
StorageInMemoryMetadata metadata
{
.columns = args.columns,
.indices = indices_description,
.constraints = args.constraints,
.partition_by_ast = partition_by_ast,
.order_by_ast = order_by_ast,
.primary_key_ast = primary_key_ast,
.sample_by_ast = sample_by_ast,
.ttl_for_table_ast = ttl_table_ast,
.settings_ast = settings_ast,
};
if (replicated)
return StorageReplicatedMergeTree::create(
zookeeper_path, replica_name, args.attach, args.database_name, args.table_name, args.relative_data_path,
args.columns, indices_description, args.constraints,
args.context, date_column_name, partition_by_ast, order_by_ast, primary_key_ast,
sample_by_ast, ttl_table_ast, settings_ast, merging_params, std::move(storage_settings),
metadata, args.context, date_column_name, merging_params, std::move(storage_settings),
args.has_force_restore_data_flag);
else
return StorageMergeTree::create(
args.database_name, args.table_name, args.relative_data_path, args.columns, indices_description,
args.constraints, args.attach, args.context, date_column_name, partition_by_ast, order_by_ast,
primary_key_ast, sample_by_ast, ttl_table_ast, settings_ast, merging_params, std::move(storage_settings),
args.database_name, args.table_name, args.relative_data_path, metadata, args.attach, args.context,
date_column_name, partition_by_ast, order_by_ast, merging_params, std::move(storage_settings),
args.has_force_restore_data_flag);
}

3
dbms/src/Storages/StorageInMemoryMetadata.h
View File

@ -35,4 +35,7 @@ struct StorageInMemoryMetadata
ASTPtr settings_ast = nullptr;
};
ASTPtr metadataToAST(const StorageInMemoryMetadata & metadata);
}

36
dbms/src/Storages/StorageMergeTree.cpp
View File

@ -56,28 +56,27 @@ StorageMergeTree::StorageMergeTree(
const String & database_name_,
const String & table_name_,
const String & relative_data_path_,
const ColumnsDescription & columns_,
const IndicesDescription & indices_,
const ConstraintsDescription & constraints_,
const StorageInMemoryMetadata & metadata,
bool attach,
Context & context_,
const String & date_column_name,
const ASTPtr & partition_by_ast_,
const ASTPtr & order_by_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported.
const ASTPtr & ttl_table_ast_,
const ASTPtr & settings_ast_,
const MergingParams & merging_params_,
std::unique_ptr<MergeTreeSettings> storage_settings_,
bool has_force_restore_data_flag)
: MergeTreeData(database_name_, table_name_, relative_data_path_,
columns_, indices_, constraints_,
context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_,
sample_by_ast_, ttl_table_ast_, settings_ast_, merging_params_,
std::move(storage_settings_), false, attach),
reader(*this), writer(*this),
merger_mutator(*this, global_context.getBackgroundPool().getNumberOfThreads())
: MergeTreeData(
database_name_,
table_name_,
relative_data_path_,
metadata,
context_,
date_column_name,
merging_params_,
std::move(storage_settings_),
false,
attach)
, reader(*this)
, writer(*this)
, merger_mutator(*this, global_context.getBackgroundPool().getNumberOfThreads())
{
loadDataParts(has_force_restore_data_flag);
@ -260,13 +259,12 @@ void StorageMergeTree::alter(
params.apply(metadata);
/// Update metdata in memory
auto update_metadata = [&]()
auto update_metadata = [&metadata, &table_lock_holder, this]()
{
changeSettings(metadata.settings_ast, table_lock_holder);
/// Reinitialize primary key because primary key column types might have changed.
setProperties(
metadata.order_by_ast, metadata.primary_key_ast, metadata.columns, metadata.indices, metadata.constraints);
setProperties(metadata);
setTTLExpressions(metadata.columns.getColumnTTLs(), metadata.ttl_for_table_ast);
};

10
dbms/src/Storages/StorageMergeTree.h
View File

@ -161,18 +161,10 @@ protected:
const String & database_name_,
const String & table_name_,
const String & relative_data_path_,
const ColumnsDescription & columns_,
const IndicesDescription & indices_,
const ConstraintsDescription & constraints_,
const StorageInMemoryMetadata & metadata,
bool attach,
Context & context_,
const String & date_column_name,
const ASTPtr & partition_by_ast_,
const ASTPtr & order_by_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported.
const ASTPtr & ttl_table_ast_,
const ASTPtr & settings_ast_,
const MergingParams & merging_params_,
std::unique_ptr<MergeTreeSettings> settings_,
bool has_force_restore_data_flag);

20
dbms/src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -195,26 +195,16 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
const String & database_name_,
const String & table_name_,
const String & relative_data_path_,
const ColumnsDescription & columns_,
const IndicesDescription & indices_,
const ConstraintsDescription & constraints_,
const StorageInMemoryMetadata & metadata,
Context & context_,
const String & date_column_name,
const ASTPtr & partition_by_ast_,
const ASTPtr & order_by_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sample_by_ast_,
const ASTPtr & ttl_table_ast_,
const ASTPtr & settings_ast_,
const MergingParams & merging_params_,
std::unique_ptr<MergeTreeSettings> settings_,
bool has_force_restore_data_flag)
: MergeTreeData(database_name_, table_name_, relative_data_path_,
columns_, indices_, constraints_,
context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_,
sample_by_ast_, ttl_table_ast_, settings_ast_, merging_params_,
std::move(settings_), true, attach,
: MergeTreeData(database_name_, table_name_, relative_data_path_, metadata,
context_, date_column_name, merging_params_, std::move(settings_), true, attach,
[this] (const std::string & name) { enqueuePartForCheck(name); }),
zookeeper_path(global_context.getMacros()->expand(zookeeper_path_, database_name_, table_name_)),
replica_name(global_context.getMacros()->expand(replica_name_, database_name_, table_name_)),
reader(*this), writer(*this), merger_mutator(*this, global_context.getBackgroundPool().getNumberOfThreads()),
@ -543,7 +533,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column
/// Even if the primary/sorting keys didn't change we must reinitialize it
/// because primary key column types might have changed.
setProperties(metadata.order_by_ast, metadata.primary_key_ast, metadata.columns, metadata.indices, metadata.constraints);
setProperties(metadata);
setTTLExpressions(new_columns.getColumnTTLs(), metadata.ttl_for_table_ast);
}

10
dbms/src/Storages/StorageReplicatedMergeTree.h
View File

@ -545,17 +545,9 @@ protected:
bool attach,
const String & database_name_, const String & name_,
const String & relative_data_path_,
const ColumnsDescription & columns_,
const IndicesDescription & indices_,
const ConstraintsDescription & constraints_,
const StorageInMemoryMetadata & metadata,
Context & context_,
const String & date_column_name,
const ASTPtr & partition_by_ast_,
const ASTPtr & order_by_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sample_by_ast_,
const ASTPtr & table_ttl_ast_,
const ASTPtr & settings_ast_,
const MergingParams & merging_params_,
std::unique_ptr<MergeTreeSettings> settings_,
bool has_force_restore_data_flag);