thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-17 11:52:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move all logic to operation

Browse Source
This commit is contained in:
Кирилл Гарбар 2024-08-01 23:57:55 +03:00
parent 102fac7b9f
commit b601d6b64d
2 changed files with 43 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/Disks/ObjectStorages/DiskObjectStorage.cpp
View File

@ -376,14 +376,9 @@ void DiskObjectStorage::removeSharedFileIfExists(const String & path, bool delet
void DiskObjectStorage::removeSharedRecursive(
const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only)
{
/// At most remove_shared_recursive_file_limit files are removed in one transaction
/// Retry until all of them are removed
while (exists(path))
{
auto transaction = createObjectStorageTransaction();
transaction->removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only);
transaction->commit();
}
auto transaction = createObjectStorageTransaction();
transaction->removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only);
transaction->commit();
}
bool DiskObjectStorage::tryReserve(UInt64 bytes)

75
src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
View File

@ -326,11 +326,46 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp
return limit > 0 && objects_to_remove_by_path.size() == limit;
}
void removeObjects()
{
if (!keep_all_batch_data)
{
std::vector<String> total_removed_paths;
total_removed_paths.reserve(objects_to_remove_by_path.size());
StoredObjects remove_from_remote;
for (auto && [local_path, objects_to_remove] : objects_to_remove_by_path)
{
chassert(!file_names_remove_metadata_only.contains(local_path));
if (objects_to_remove.unlink_outcome->num_hardlinks == 0)
{
std::move(objects_to_remove.objects.begin(), objects_to_remove.objects.end(), std::back_inserter(remove_from_remote));
total_removed_paths.push_back(local_path);
}
}
/// Read comment inside RemoveObjectStorageOperation class
/// TL;DR Don't pay any attention to 404 status code
object_storage.removeObjectsIfExist(remove_from_remote);
objects_to_remove_by_path.clear();
LOG_DEBUG(
getLogger("RemoveRecursiveObjectStorageOperation"),
"Recursively remove path {}: "
"metadata and objects were removed for [{}], "
"only metadata were removed for [{}].",
path,
boost::algorithm::join(total_removed_paths, ", "),
boost::algorithm::join(file_names_remove_metadata_only, ", "));
}
}
void removeMetadataRecursive(MetadataTransactionPtr tx, const std::string & path_to_remove)
{
checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks.
if (checkLimitReached())
return;
removeObjects();
if (metadata_storage.isFile(path_to_remove))
{
@ -373,13 +408,11 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp
{
for (auto it = metadata_storage.iterateDirectory(path_to_remove); it->isValid(); it->next())
{
if (checkLimitReached())
return;
removeMetadataRecursive(tx, it->path());
if (checkLimitReached())
removeObjects();
}
/// Do not delete in case directory contains >= limit files
if (objects_to_remove_by_path.size() < limit)
tx->removeDirectory(path_to_remove);
tx->removeDirectory(path_to_remove);
}
}
@ -396,35 +429,7 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp
void finalize() override
{
if (!keep_all_batch_data)
{
std::vector<String> total_removed_paths;
total_removed_paths.reserve(objects_to_remove_by_path.size());
StoredObjects remove_from_remote;
for (auto && [local_path, objects_to_remove] : objects_to_remove_by_path)
{
chassert(!file_names_remove_metadata_only.contains(local_path));
if (objects_to_remove.unlink_outcome->num_hardlinks == 0)
{
std::move(objects_to_remove.objects.begin(), objects_to_remove.objects.end(), std::back_inserter(remove_from_remote));
total_removed_paths.push_back(local_path);
}
}
/// Read comment inside RemoveObjectStorageOperation class
/// TL;DR Don't pay any attention to 404 status code
object_storage.removeObjectsIfExist(remove_from_remote);
LOG_DEBUG(
getLogger("RemoveRecursiveObjectStorageOperation"),
"Recursively remove path {}: "
"metadata and objects were removed for [{}], "
"only metadata were removed for [{}].",
path,
boost::algorithm::join(total_removed_paths, ", "),
boost::algorithm::join(file_names_remove_metadata_only, ", "));
}
removeObjects();
}
};