mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-24 08:32:02 +00:00
Merge pull request #21142 from Jokser/unfreeze-partitions
Add ALTER TABLE UNFREEZE command.
This commit is contained in:
commit
36898bdc4a
@ -63,7 +63,7 @@ enum class AccessType
|
||||
M(ALTER_SETTINGS, "ALTER SETTING, ALTER MODIFY SETTING, MODIFY SETTING", TABLE, ALTER_TABLE) /* allows to execute ALTER MODIFY SETTING */\
|
||||
M(ALTER_MOVE_PARTITION, "ALTER MOVE PART, MOVE PARTITION, MOVE PART", TABLE, ALTER_TABLE) \
|
||||
M(ALTER_FETCH_PARTITION, "FETCH PARTITION", TABLE, ALTER_TABLE) \
|
||||
M(ALTER_FREEZE_PARTITION, "FREEZE PARTITION", TABLE, ALTER_TABLE) \
|
||||
M(ALTER_FREEZE_PARTITION, "FREEZE PARTITION, UNFREEZE", TABLE, ALTER_TABLE) \
|
||||
\
|
||||
M(ALTER_TABLE, "", GROUP, ALTER) \
|
||||
\
|
||||
|
@ -299,7 +299,9 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand(const AS
|
||||
break;
|
||||
}
|
||||
case ASTAlterCommand::FREEZE_PARTITION: [[fallthrough]];
|
||||
case ASTAlterCommand::FREEZE_ALL:
|
||||
case ASTAlterCommand::FREEZE_ALL: [[fallthrough]];
|
||||
case ASTAlterCommand::UNFREEZE_PARTITION: [[fallthrough]];
|
||||
case ASTAlterCommand::UNFREEZE_ALL:
|
||||
{
|
||||
required_access.emplace_back(AccessType::ALTER_FREEZE_PARTITION, database, table);
|
||||
break;
|
||||
|
@ -271,6 +271,27 @@ void ASTAlterCommand::formatImpl(
|
||||
<< " " << DB::quote << with_name;
|
||||
}
|
||||
}
|
||||
else if (type == ASTAlterCommand::UNFREEZE_PARTITION)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "UNFREEZE PARTITION " << (settings.hilite ? hilite_none : "");
|
||||
partition->formatImpl(settings, state, frame);
|
||||
|
||||
if (!with_name.empty())
|
||||
{
|
||||
settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "")
|
||||
<< " " << DB::quote << with_name;
|
||||
}
|
||||
}
|
||||
else if (type == ASTAlterCommand::UNFREEZE_ALL)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "UNFREEZE";
|
||||
|
||||
if (!with_name.empty())
|
||||
{
|
||||
settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << "WITH NAME" << (settings.hilite ? hilite_none : "")
|
||||
<< " " << DB::quote << with_name;
|
||||
}
|
||||
}
|
||||
else if (type == ASTAlterCommand::DELETE)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "DELETE" << (settings.hilite ? hilite_none : "");
|
||||
@ -368,7 +389,8 @@ bool ASTAlterQuery::isSettingsAlter() const
|
||||
|
||||
bool ASTAlterQuery::isFreezeAlter() const
|
||||
{
|
||||
return isOneCommandTypeOnly(ASTAlterCommand::FREEZE_PARTITION) || isOneCommandTypeOnly(ASTAlterCommand::FREEZE_ALL);
|
||||
return isOneCommandTypeOnly(ASTAlterCommand::FREEZE_PARTITION) || isOneCommandTypeOnly(ASTAlterCommand::FREEZE_ALL)
|
||||
|| isOneCommandTypeOnly(ASTAlterCommand::UNFREEZE_PARTITION) || isOneCommandTypeOnly(ASTAlterCommand::UNFREEZE_ALL);
|
||||
}
|
||||
|
||||
/** Get the text that identifies this element. */
|
||||
|
@ -54,6 +54,8 @@ public:
|
||||
FETCH_PARTITION,
|
||||
FREEZE_PARTITION,
|
||||
FREEZE_ALL,
|
||||
UNFREEZE_PARTITION,
|
||||
UNFREEZE_ALL,
|
||||
|
||||
DELETE,
|
||||
UPDATE,
|
||||
@ -153,7 +155,9 @@ public:
|
||||
*/
|
||||
String from;
|
||||
|
||||
/** For FREEZE PARTITION - place local backup to directory with specified name.
|
||||
/**
|
||||
* For FREEZE PARTITION - place local backup to directory with specified name.
|
||||
* For UNFREEZE - delete local backup at directory with specified name.
|
||||
*/
|
||||
String with_name;
|
||||
|
||||
|
@ -63,6 +63,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
ParserKeyword s_fetch_partition("FETCH PARTITION");
|
||||
ParserKeyword s_replace_partition("REPLACE PARTITION");
|
||||
ParserKeyword s_freeze("FREEZE");
|
||||
ParserKeyword s_unfreeze("UNFREEZE");
|
||||
ParserKeyword s_partition("PARTITION");
|
||||
|
||||
ParserKeyword s_first("FIRST");
|
||||
@ -454,6 +455,37 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
|
||||
command->with_name = ast_with_name->as<ASTLiteral &>().value.get<const String &>();
|
||||
}
|
||||
}
|
||||
else if (s_unfreeze.ignore(pos, expected))
|
||||
{
|
||||
if (s_partition.ignore(pos, expected))
|
||||
{
|
||||
if (!parser_partition.parse(pos, command->partition, expected))
|
||||
return false;
|
||||
|
||||
command->type = ASTAlterCommand::UNFREEZE_PARTITION;
|
||||
}
|
||||
else
|
||||
{
|
||||
command->type = ASTAlterCommand::UNFREEZE_ALL;
|
||||
}
|
||||
|
||||
/// WITH NAME 'name' - remove local backup to directory with specified name
|
||||
if (s_with.ignore(pos, expected))
|
||||
{
|
||||
if (!s_name.ignore(pos, expected))
|
||||
return false;
|
||||
|
||||
ASTPtr ast_with_name;
|
||||
if (!parser_string_literal.parse(pos, ast_with_name, expected))
|
||||
return false;
|
||||
|
||||
command->with_name = ast_with_name->as<ASTLiteral &>().value.get<const String &>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else if (s_modify_column.ignore(pos, expected))
|
||||
{
|
||||
if (s_if_exists.ignore(pos, expected))
|
||||
|
@ -1849,11 +1849,6 @@ void MergeTreeData::changeSettings(
|
||||
}
|
||||
}
|
||||
|
||||
PartitionCommandsResultInfo MergeTreeData::freezeAll(const String & with_name, const StorageMetadataPtr & metadata_snapshot, const Context & context, TableLockHolder &)
|
||||
{
|
||||
return freezePartitionsByMatcher([] (const DataPartPtr &) { return true; }, metadata_snapshot, with_name, context);
|
||||
}
|
||||
|
||||
void MergeTreeData::PartsTemporaryRename::addPart(const String & old_name, const String & new_name)
|
||||
{
|
||||
old_and_new_names.push_back({old_name, new_name});
|
||||
@ -2690,44 +2685,6 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
PartitionCommandsResultInfo MergeTreeData::freezePartition(const ASTPtr & partition_ast, const StorageMetadataPtr & metadata_snapshot, const String & with_name, const Context & context, TableLockHolder &)
|
||||
{
|
||||
std::optional<String> prefix;
|
||||
String partition_id;
|
||||
|
||||
if (format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
|
||||
{
|
||||
/// Month-partitioning specific - partition value can represent a prefix of the partition to freeze.
|
||||
if (const auto * partition_lit = partition_ast->as<ASTPartition &>().value->as<ASTLiteral>())
|
||||
prefix = partition_lit->value.getType() == Field::Types::UInt64
|
||||
? toString(partition_lit->value.get<UInt64>())
|
||||
: partition_lit->value.safeGet<String>();
|
||||
else
|
||||
partition_id = getPartitionIDFromQuery(partition_ast, context);
|
||||
}
|
||||
else
|
||||
partition_id = getPartitionIDFromQuery(partition_ast, context);
|
||||
|
||||
if (prefix)
|
||||
LOG_DEBUG(log, "Freezing parts with prefix {}", *prefix);
|
||||
else
|
||||
LOG_DEBUG(log, "Freezing parts with partition ID {}", partition_id);
|
||||
|
||||
|
||||
return freezePartitionsByMatcher(
|
||||
[&prefix, &partition_id](const DataPartPtr & part)
|
||||
{
|
||||
if (prefix)
|
||||
return startsWith(part->info.partition_id, *prefix);
|
||||
else
|
||||
return part->info.partition_id == partition_id;
|
||||
},
|
||||
metadata_snapshot,
|
||||
with_name,
|
||||
context);
|
||||
}
|
||||
|
||||
void MergeTreeData::checkAlterPartitionIsPossible(const PartitionCommands & commands, const StorageMetadataPtr & /*metadata_snapshot*/, const Settings & settings) const
|
||||
{
|
||||
for (const auto & command : commands)
|
||||
@ -2957,6 +2914,21 @@ Pipe MergeTreeData::alterPartition(
|
||||
current_command_results = freezeAll(command.with_name, metadata_snapshot, query_context, lock);
|
||||
}
|
||||
break;
|
||||
|
||||
case PartitionCommand::UNFREEZE_PARTITION:
|
||||
{
|
||||
auto lock = lockForShare(query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout);
|
||||
current_command_results = unfreezePartition(command.partition, command.with_name, query_context, lock);
|
||||
}
|
||||
break;
|
||||
|
||||
case PartitionCommand::UNFREEZE_ALL_PARTITIONS:
|
||||
{
|
||||
auto lock = lockForShare(query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout);
|
||||
current_command_results = unfreezeAll(command.with_name, query_context, lock);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
for (auto & command_result : current_command_results)
|
||||
command_result.command_type = command.typeToString();
|
||||
@ -3719,7 +3691,60 @@ MergeTreeData::PathsWithDisks MergeTreeData::getRelativeDataPathsWithDisks() con
|
||||
return res;
|
||||
}
|
||||
|
||||
PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const StorageMetadataPtr & metadata_snapshot, const String & with_name, const Context & context)
|
||||
MergeTreeData::MatcherFn MergeTreeData::getPartitionMatcher(const ASTPtr & partition_ast, const Context & context) const
|
||||
{
|
||||
bool prefixed = false;
|
||||
String id;
|
||||
|
||||
if (format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
|
||||
{
|
||||
/// Month-partitioning specific - partition value can represent a prefix of the partition to freeze.
|
||||
if (const auto * partition_lit = partition_ast->as<ASTPartition &>().value->as<ASTLiteral>())
|
||||
{
|
||||
id = partition_lit->value.getType() == Field::Types::UInt64
|
||||
? toString(partition_lit->value.get<UInt64>())
|
||||
: partition_lit->value.safeGet<String>();
|
||||
prefixed = true;
|
||||
}
|
||||
else
|
||||
id = getPartitionIDFromQuery(partition_ast, context);
|
||||
}
|
||||
else
|
||||
id = getPartitionIDFromQuery(partition_ast, context);
|
||||
|
||||
return [prefixed, id](const String & partition_id)
|
||||
{
|
||||
if (prefixed)
|
||||
return startsWith(partition_id, id);
|
||||
else
|
||||
return id == partition_id;
|
||||
};
|
||||
}
|
||||
|
||||
PartitionCommandsResultInfo MergeTreeData::freezePartition(
|
||||
const ASTPtr & partition_ast,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const String & with_name,
|
||||
const Context & context,
|
||||
TableLockHolder &)
|
||||
{
|
||||
return freezePartitionsByMatcher(getPartitionMatcher(partition_ast, context), metadata_snapshot, with_name, context);
|
||||
}
|
||||
|
||||
PartitionCommandsResultInfo MergeTreeData::freezeAll(
|
||||
const String & with_name,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const Context & context,
|
||||
TableLockHolder &)
|
||||
{
|
||||
return freezePartitionsByMatcher([] (const String &) { return true; }, metadata_snapshot, with_name, context);
|
||||
}
|
||||
|
||||
PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(
|
||||
MatcherFn matcher,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const String & with_name,
|
||||
const Context & context)
|
||||
{
|
||||
String clickhouse_path = Poco::Path(context.getPath()).makeAbsolute().toString();
|
||||
String default_shadow_path = clickhouse_path + "shadow/";
|
||||
@ -3742,7 +3767,7 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(MatcherFn m
|
||||
size_t parts_processed = 0;
|
||||
for (const auto & part : data_parts)
|
||||
{
|
||||
if (!matcher(part))
|
||||
if (!matcher(part->info.partition_id))
|
||||
continue;
|
||||
|
||||
LOG_DEBUG(log, "Freezing part {} snapshot will be placed at {}", part->name, backup_path);
|
||||
@ -3772,6 +3797,70 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher(MatcherFn m
|
||||
return result;
|
||||
}
|
||||
|
||||
PartitionCommandsResultInfo MergeTreeData::unfreezePartition(
|
||||
const ASTPtr & partition,
|
||||
const String & backup_name,
|
||||
const Context & context,
|
||||
TableLockHolder &)
|
||||
{
|
||||
return unfreezePartitionsByMatcher(getPartitionMatcher(partition, context), backup_name, context);
|
||||
}
|
||||
|
||||
PartitionCommandsResultInfo MergeTreeData::unfreezeAll(
|
||||
const String & backup_name,
|
||||
const Context & context,
|
||||
TableLockHolder &)
|
||||
{
|
||||
return unfreezePartitionsByMatcher([] (const String &) { return true; }, backup_name, context);
|
||||
}
|
||||
|
||||
PartitionCommandsResultInfo MergeTreeData::unfreezePartitionsByMatcher(MatcherFn matcher, const String & backup_name, const Context &)
|
||||
{
|
||||
auto backup_path = std::filesystem::path("shadow") / escapeForFileName(backup_name) / relative_data_path;
|
||||
|
||||
LOG_DEBUG(log, "Unfreezing parts by path {}", backup_path.generic_string());
|
||||
|
||||
PartitionCommandsResultInfo result;
|
||||
|
||||
for (const auto & disk : getStoragePolicy()->getDisks())
|
||||
{
|
||||
if (!disk->exists(backup_path))
|
||||
continue;
|
||||
|
||||
for (auto it = disk->iterateDirectory(backup_path); it->isValid(); it->next())
|
||||
{
|
||||
const auto & partition_directory = it->name();
|
||||
|
||||
/// Partition ID is prefix of part directory name: <partition id>_<rest of part directory name>
|
||||
auto found = partition_directory.find('_');
|
||||
if (found == std::string::npos)
|
||||
continue;
|
||||
auto partition_id = partition_directory.substr(0, found);
|
||||
|
||||
if (!matcher(partition_id))
|
||||
continue;
|
||||
|
||||
const auto & path = it->path();
|
||||
|
||||
disk->removeRecursive(path);
|
||||
|
||||
result.push_back(PartitionCommandResultInfo{
|
||||
.partition_id = partition_id,
|
||||
.part_name = partition_directory,
|
||||
.backup_path = disk->getPath() + backup_path.generic_string(),
|
||||
.part_backup_path = disk->getPath() + path,
|
||||
.backup_name = backup_name,
|
||||
});
|
||||
|
||||
LOG_DEBUG(log, "Unfreezed part by path {}", disk->getPath() + path);
|
||||
}
|
||||
}
|
||||
|
||||
LOG_DEBUG(log, "Unfreezed {} parts", result.size());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
bool MergeTreeData::canReplacePartition(const DataPartPtr & src_part) const
|
||||
{
|
||||
const auto settings = getSettings();
|
||||
|
@ -544,13 +544,6 @@ public:
|
||||
const ASTPtr & new_settings,
|
||||
TableLockHolder & table_lock_holder);
|
||||
|
||||
/// Freezes all parts.
|
||||
PartitionCommandsResultInfo freezeAll(
|
||||
const String & with_name,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const Context & context,
|
||||
TableLockHolder & table_lock_holder);
|
||||
|
||||
/// Should be called if part data is suspected to be corrupted.
|
||||
void reportBrokenPart(const String & name) const
|
||||
{
|
||||
@ -568,8 +561,32 @@ public:
|
||||
* Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number,
|
||||
* or if 'with_name' is specified - backup is created in directory with specified name.
|
||||
*/
|
||||
PartitionCommandsResultInfo freezePartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, const String & with_name, const Context & context, TableLockHolder & table_lock_holder);
|
||||
PartitionCommandsResultInfo freezePartition(
|
||||
const ASTPtr & partition,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const String & with_name,
|
||||
const Context & context,
|
||||
TableLockHolder & table_lock_holder);
|
||||
|
||||
/// Freezes all parts.
|
||||
PartitionCommandsResultInfo freezeAll(
|
||||
const String & with_name,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const Context & context,
|
||||
TableLockHolder & table_lock_holder);
|
||||
|
||||
/// Unfreezes particular partition.
|
||||
PartitionCommandsResultInfo unfreezePartition(
|
||||
const ASTPtr & partition,
|
||||
const String & backup_name,
|
||||
const Context & context,
|
||||
TableLockHolder & table_lock_holder);
|
||||
|
||||
/// Unfreezes all parts.
|
||||
PartitionCommandsResultInfo unfreezeAll(
|
||||
const String & backup_name,
|
||||
const Context & context,
|
||||
TableLockHolder & table_lock_holder);
|
||||
|
||||
public:
|
||||
/// Moves partition to specified Disk
|
||||
@ -939,8 +956,9 @@ protected:
|
||||
bool isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const ASTPtr & node, const StorageMetadataPtr & metadata_snapshot) const;
|
||||
|
||||
/// Common part for |freezePartition()| and |freezeAll()|.
|
||||
using MatcherFn = std::function<bool(const DataPartPtr &)>;
|
||||
using MatcherFn = std::function<bool(const String &)>;
|
||||
PartitionCommandsResultInfo freezePartitionsByMatcher(MatcherFn matcher, const StorageMetadataPtr & metadata_snapshot, const String & with_name, const Context & context);
|
||||
PartitionCommandsResultInfo unfreezePartitionsByMatcher(MatcherFn matcher, const String & backup_name, const Context & context);
|
||||
|
||||
// Partition helpers
|
||||
bool canReplacePartition(const DataPartPtr & src_part) const;
|
||||
@ -1024,6 +1042,9 @@ private:
|
||||
// Record all query ids which access the table. It's guarded by `query_id_set_mutex` and is always mutable.
|
||||
mutable std::set<String> query_id_set;
|
||||
mutable std::mutex query_id_set_mutex;
|
||||
|
||||
// Get partition matcher for FREEZE / UNFREEZE queries.
|
||||
MatcherFn getPartitionMatcher(const ASTPtr & partition, const Context & context) const;
|
||||
};
|
||||
|
||||
/// RAII struct to record big parts that are submerging or emerging.
|
||||
|
@ -94,10 +94,25 @@ std::optional<PartitionCommand> PartitionCommand::parse(const ASTAlterCommand *
|
||||
}
|
||||
else if (command_ast->type == ASTAlterCommand::FREEZE_ALL)
|
||||
{
|
||||
PartitionCommand command;
|
||||
command.type = PartitionCommand::FREEZE_ALL_PARTITIONS;
|
||||
command.with_name = command_ast->with_name;
|
||||
return command;
|
||||
PartitionCommand res;
|
||||
res.type = PartitionCommand::FREEZE_ALL_PARTITIONS;
|
||||
res.with_name = command_ast->with_name;
|
||||
return res;
|
||||
}
|
||||
else if (command_ast->type == ASTAlterCommand::UNFREEZE_PARTITION)
|
||||
{
|
||||
PartitionCommand res;
|
||||
res.type = PartitionCommand::UNFREEZE_PARTITION;
|
||||
res.partition = command_ast->partition;
|
||||
res.with_name = command_ast->with_name;
|
||||
return res;
|
||||
}
|
||||
else if (command_ast->type == ASTAlterCommand::UNFREEZE_ALL)
|
||||
{
|
||||
PartitionCommand res;
|
||||
res.type = PartitionCommand::UNFREEZE_ALL_PARTITIONS;
|
||||
res.with_name = command_ast->with_name;
|
||||
return res;
|
||||
}
|
||||
else
|
||||
return {};
|
||||
@ -130,6 +145,10 @@ std::string PartitionCommand::typeToString() const
|
||||
return "FREEZE ALL";
|
||||
case PartitionCommand::Type::FREEZE_PARTITION:
|
||||
return "FREEZE PARTITION";
|
||||
case PartitionCommand::Type::UNFREEZE_PARTITION:
|
||||
return "UNFREEZE PARTITION";
|
||||
case PartitionCommand::Type::UNFREEZE_ALL_PARTITIONS:
|
||||
return "UNFREEZE ALL";
|
||||
case PartitionCommand::Type::REPLACE_PARTITION:
|
||||
return "REPLACE PARTITION";
|
||||
}
|
||||
|
@ -27,6 +27,8 @@ struct PartitionCommand
|
||||
FETCH_PARTITION,
|
||||
FREEZE_ALL_PARTITIONS,
|
||||
FREEZE_PARTITION,
|
||||
UNFREEZE_ALL_PARTITIONS,
|
||||
UNFREEZE_PARTITION,
|
||||
REPLACE_PARTITION,
|
||||
};
|
||||
|
||||
@ -52,7 +54,7 @@ struct PartitionCommand
|
||||
/// For FETCH PARTITION - path in ZK to the shard, from which to download the partition.
|
||||
String from_zookeeper_path;
|
||||
|
||||
/// For FREEZE PARTITION
|
||||
/// For FREEZE PARTITION and UNFREEZE
|
||||
String with_name;
|
||||
|
||||
enum MoveDestinationType
|
||||
|
@ -137,8 +137,17 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical):
|
||||
list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD_PER_PART_WIDE * 6 + FILES_OVERHEAD
|
||||
|
||||
node.query("SYSTEM START MERGES s3_test")
|
||||
|
||||
# Wait for merges and old parts deletion
|
||||
time.sleep(3)
|
||||
for attempt in range(0, 10):
|
||||
parts_count = node.query("SELECT COUNT(*) FROM system.parts WHERE table = 's3_test' FORMAT Values")
|
||||
if parts_count == "(1)":
|
||||
break
|
||||
|
||||
if attempt == 9:
|
||||
assert parts_count == "(1)"
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)"
|
||||
assert node.query("SELECT count(distinct(id)) FROM s3_test FORMAT Values") == "(8192)"
|
||||
@ -333,3 +342,28 @@ def test_move_replace_partition_to_another_table(cluster):
|
||||
|
||||
for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')):
|
||||
minio.remove_object(cluster.minio_bucket, obj.object_name)
|
||||
|
||||
|
||||
def test_freeze_unfreeze(cluster):
|
||||
create_table(cluster, "s3_test")
|
||||
|
||||
node = cluster.instances["node"]
|
||||
minio = cluster.minio_client
|
||||
|
||||
node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096)))
|
||||
node.query("ALTER TABLE s3_test FREEZE WITH NAME 'backup1'")
|
||||
node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-04', 4096)))
|
||||
node.query("ALTER TABLE s3_test FREEZE WITH NAME 'backup2'")
|
||||
|
||||
node.query("TRUNCATE TABLE s3_test")
|
||||
assert len(
|
||||
list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2
|
||||
|
||||
# Unfreeze single partition from backup1.
|
||||
node.query("ALTER TABLE s3_test UNFREEZE PARTITION '2020-01-03' WITH NAME 'backup1'")
|
||||
# Unfreeze all partitions from backup2.
|
||||
node.query("ALTER TABLE s3_test UNFREEZE WITH NAME 'backup2'")
|
||||
|
||||
# Data should be removed from S3.
|
||||
assert len(
|
||||
list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD
|
||||
|
@ -29,7 +29,7 @@ ALTER MATERIALIZE TTL ['MATERIALIZE TTL'] TABLE ALTER TABLE
|
||||
ALTER SETTINGS ['ALTER SETTING','ALTER MODIFY SETTING','MODIFY SETTING'] TABLE ALTER TABLE
|
||||
ALTER MOVE PARTITION ['ALTER MOVE PART','MOVE PARTITION','MOVE PART'] TABLE ALTER TABLE
|
||||
ALTER FETCH PARTITION ['FETCH PARTITION'] TABLE ALTER TABLE
|
||||
ALTER FREEZE PARTITION ['FREEZE PARTITION'] TABLE ALTER TABLE
|
||||
ALTER FREEZE PARTITION ['FREEZE PARTITION','UNFREEZE'] TABLE ALTER TABLE
|
||||
ALTER TABLE [] \N ALTER
|
||||
ALTER VIEW REFRESH ['ALTER LIVE VIEW REFRESH','REFRESH VIEW'] VIEW ALTER VIEW
|
||||
ALTER VIEW MODIFY QUERY ['ALTER TABLE MODIFY QUERY'] VIEW ALTER VIEW
|
||||
|
@ -16,3 +16,9 @@ ATTACH PARTITION 3 3_12_12_0 3_4_4_0
|
||||
command_type partition_id part_name backup_name old_part_name
|
||||
FREEZE PARTITION 7 7_8_8_0 test_01417_single_part_7
|
||||
ATTACH PART 5 5_13_13_0 5_6_6_0
|
||||
command_type partition_id part_name backup_name
|
||||
UNFREEZE PARTITION 7 7_8_8_0 test_01417_single_part_7
|
||||
command_type partition_id part_name backup_name
|
||||
FREEZE PARTITION 202103 20210301_20210301_1_1_0 test_01417_single_part_old_syntax
|
||||
command_type partition_id part_name backup_name
|
||||
UNFREEZE PARTITION 20210301 20210301_20210301_1_1_0 test_01417_single_part_old_syntax
|
||||
|
@ -13,6 +13,11 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze;"
|
||||
${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze (key UInt64, value String) ENGINE = MergeTree() ORDER BY key PARTITION BY key % 10;"
|
||||
${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze SELECT number, toString(number) from numbers(10);"
|
||||
|
||||
# also for old syntax
|
||||
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze_old_syntax;"
|
||||
${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze_old_syntax (dt Date, value String) ENGINE = MergeTree(dt, (value), 8192);"
|
||||
${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze_old_syntax SELECT toDate('2021-03-01'), toString(number) from numbers(10);"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
|
||||
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \
|
||||
--query "SELECT command_type, partition_id, part_name, backup_name FROM table"
|
||||
@ -35,5 +40,21 @@ ${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze FREEZE PARTITION '7'
|
||||
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE, $ATTACH_OUT_STRUCTURE" \
|
||||
--query "SELECT command_type, partition_id, part_name, backup_name, old_part_name FROM table"
|
||||
|
||||
# Unfreeze partition
|
||||
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze UNFREEZE PARTITION '7' WITH NAME 'test_01417_single_part_7' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
|
||||
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \
|
||||
--query "SELECT command_type, partition_id, part_name, backup_name FROM table"
|
||||
|
||||
# Freeze partition with old syntax
|
||||
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_old_syntax FREEZE PARTITION '202103' WITH NAME 'test_01417_single_part_old_syntax' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
|
||||
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \
|
||||
--query "SELECT command_type, partition_id, part_name, backup_name FROM table"
|
||||
|
||||
# Unfreeze partition with old syntax
|
||||
${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_old_syntax UNFREEZE PARTITION '202103' WITH NAME 'test_01417_single_part_old_syntax' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \
|
||||
| ${CLICKHOUSE_LOCAL} --structure "$ALTER_OUT_STRUCTURE, $FREEZE_OUT_STRUCTURE" \
|
||||
--query "SELECT command_type, partition_id, part_name, backup_name FROM table"
|
||||
|
||||
# teardown
|
||||
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze;"
|
||||
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze_old_syntax;"
|
||||
|
@ -1,6 +1,6 @@
|
||||
AlterQuery t1 (children 1)
|
||||
ExpressionList (children 1)
|
||||
AlterCommand 25 (children 1)
|
||||
AlterCommand 27 (children 1)
|
||||
Function equals (children 1)
|
||||
ExpressionList (children 2)
|
||||
Identifier date
|
||||
|
Loading…
Reference in New Issue
Block a user