thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix alter modify query for partition key and other metadata fields

Browse Source
This commit is contained in:
alesapin 2021-03-01 12:59:19 +03:00
parent 11f2a271a2
commit 9c8afbeb53
4 changed files with 119 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -165,7 +165,7 @@ MergeTreeData::MergeTreeData(
{
try
{
checkPartitionKeyAndInitMinMax(metadata_.partition_key);
setProperties(metadata_, metadata_, attach);
if (minmax_idx_date_column_pos == -1)
throw Exception("Could not find Date column", ErrorCodes::BAD_TYPE_OF_FIELD);
}
@ -179,12 +179,10 @@ MergeTreeData::MergeTreeData(
else
{
is_custom_partitioned = true;
checkPartitionKeyAndInitMinMax(metadata_.partition_key);
setProperties(metadata_, metadata_, attach);
min_format_version = MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING;
}
setProperties(metadata_, metadata_, attach);
/// NOTE: using the same columns list as is read when performing actual merges.
merging_params.check(metadata_);
@ -398,6 +396,7 @@ void MergeTreeData::checkProperties(
void MergeTreeData::setProperties(const StorageInMemoryMetadata & new_metadata, const StorageInMemoryMetadata & old_metadata, bool attach)
{
checkProperties(new_metadata, old_metadata, attach);
checkPartitionKeyAndInitMinMax(new_metadata.partition_key);
setInMemoryMetadata(new_metadata);
}
@ -440,6 +439,12 @@ void MergeTreeData::checkPartitionKeyAndInitMinMax(const KeyDescription & new_pa
checkKeyExpression(*new_partition_key.expression, new_partition_key.sample_block, "Partition", allow_nullable_key);
/// Reset filled fields
minmax_idx_columns.clear();
minmax_idx_column_types.clear();
minmax_idx_date_column_pos = -1;
minmax_idx_time_column_pos = -1;
/// Add all columns used in the partition key to the min-max index.
const NamesAndTypesList & minmax_idx_columns_with_types = new_partition_key.expression->getRequiredColumnsWithTypes();
minmax_idx_expr = std::make_shared<ExpressionActions>(std::make_shared<ActionsDAG>(minmax_idx_columns_with_types));

54
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -897,21 +897,8 @@ void StorageReplicatedMergeTree::setTableStructure(
StorageInMemoryMetadata old_metadata = getInMemoryMetadata();
if (new_columns != new_metadata.columns)
{
new_metadata.columns = new_columns;
new_metadata.column_ttls_by_name.clear();
for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs())
{
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, global_context, new_metadata.primary_key);
new_metadata.column_ttls_by_name[name] = new_ttl_entry;
}
/// The type of partition key expression may change
if (new_metadata.partition_key.definition_ast != nullptr)
new_metadata.partition_key.recalculateWithNewColumns(new_metadata.columns, global_context);
}
if (!metadata_diff.empty())
{
auto parse_key_expr = [] (const String & key_expr)
@ -977,6 +964,47 @@ void StorageReplicatedMergeTree::setTableStructure(
}
}
/// Changes in columns may affect following metadata fields
if (new_metadata.columns != old_metadata.columns)
{
new_metadata.column_ttls_by_name.clear();
for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs())
{
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, global_context, new_metadata.primary_key);
new_metadata.column_ttls_by_name[name] = new_ttl_entry;
}
if (new_metadata.partition_key.definition_ast != nullptr)
new_metadata.partition_key.recalculateWithNewColumns(new_metadata.columns, global_context);
if (!metadata_diff.sorting_key_changed) /// otherwise already updated
new_metadata.sorting_key.recalculateWithNewColumns(new_metadata.columns, global_context);
/// Primary key is special, it exists even if not defined
if (new_metadata.primary_key.definition_ast != nullptr)
{
new_metadata.primary_key.recalculateWithNewColumns(new_metadata.columns, global_context);
}
else
{
new_metadata.primary_key = KeyDescription::getKeyFromAST(new_metadata.sorting_key.definition_ast, new_metadata.columns, global_context);
new_metadata.primary_key.definition_ast = nullptr;
}
if (!metadata_diff.sampling_expression_changed && new_metadata.sampling_key.definition_ast != nullptr)
new_metadata.sampling_key.recalculateWithNewColumns(new_metadata.columns, global_context);
if (!metadata_diff.skip_indices_changed) /// otherwise already updated
{
for (auto & index : new_metadata.secondary_indices)
index.recalculateWithNewColumns(new_metadata.columns, global_context);
}
if (!metadata_diff.ttl_table_changed && new_metadata.table_ttl.definition_ast != nullptr)
new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(
new_metadata.table_ttl.definition_ast, new_metadata.columns, global_context, new_metadata.primary_key);
}
/// Even if the primary/sorting/partition keys didn't change we must reinitialize it
/// because primary/partition key column types might have changed.
checkTTLExpressions(new_metadata, old_metadata);

6
tests/queries/0_stateless/01747_alter_partition_key_enum_zookeeper.reference Normal file
View File

@ -0,0 +1,6 @@
IU lada 2101 1970-04-19 15:00:00
PS jeep Grand Cherokee 2005-10-03 15:00:00
PS jeep Grand Cherokee 2005-10-03 15:00:00
IU lada 2101 1970-04-19 15:00:00
PS jeep Grand Cherokee 2005-10-03 15:00:00
PS jeep Grand Cherokee 2005-10-03 15:00:00

63
tests/queries/0_stateless/01747_alter_partition_key_enum_zookeeper.sql Normal file
View File

@ -0,0 +1,63 @@
DROP TABLE IF EXISTS report;
CREATE TABLE report
(
`product` Enum8('IU' = 1, 'WS' = 2),
`machine` String,
`branch` String,
`generated_time` DateTime
)
ENGINE = MergeTree
PARTITION BY (product, toYYYYMM(generated_time))
ORDER BY (product, machine, branch, generated_time);
INSERT INTO report VALUES ('IU', 'lada', '2101', toDateTime('1970-04-19 15:00:00'));
SELECT * FROM report WHERE product = 'IU';
ALTER TABLE report MODIFY COLUMN product Enum8('IU' = 1, 'WS' = 2, 'PS' = 3);
SELECT * FROM report WHERE product = 'PS';
INSERT INTO report VALUES ('PS', 'jeep', 'Grand Cherokee', toDateTime('2005-10-03 15:00:00'));
SELECT * FROM report WHERE product = 'PS';
DETACH TABLE report;
ATTACH TABLE report;
SELECT * FROM report WHERE product = 'PS';
DROP TABLE IF EXISTS report;
DROP TABLE IF EXISTS replicated_report;
CREATE TABLE replicated_report
(
`product` Enum8('IU' = 1, 'WS' = 2),
`machine` String,
`branch` String,
`generated_time` DateTime
)
ENGINE = ReplicatedMergeTree('/clickhouse/01747_alter_partition_key/t', '1')
PARTITION BY (product, toYYYYMM(generated_time))
ORDER BY (product, machine, branch, generated_time);
INSERT INTO replicated_report VALUES ('IU', 'lada', '2101', toDateTime('1970-04-19 15:00:00'));
SELECT * FROM replicated_report WHERE product = 'IU';
ALTER TABLE replicated_report MODIFY COLUMN product Enum8('IU' = 1, 'WS' = 2, 'PS' = 3) SETTINGS replication_alter_partitions_sync=2;
SELECT * FROM replicated_report WHERE product = 'PS';
INSERT INTO replicated_report VALUES ('PS', 'jeep', 'Grand Cherokee', toDateTime('2005-10-03 15:00:00'));
SELECT * FROM replicated_report WHERE product = 'PS';
DETACH TABLE replicated_report;
ATTACH TABLE replicated_report;
SELECT * FROM replicated_report WHERE product = 'PS';
DROP TABLE IF EXISTS replicated_report;