thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-28 02:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Check refcount in finalize for other RemoveObjectStorageOperations

Browse Source
This commit is contained in:
vdimir 2023-07-07 16:42:03 +00:00
parent 227e415d6d
commit 88911e1378
No known key found for this signature in database
GPG Key ID: 6EE4CE2BEDC51862
1 changed files with 24 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
View File

@ -65,11 +65,18 @@ struct PureMetadataObjectStorageOperation final : public IDiskObjectStorageOpera
std::string getInfoForLog() const override { return fmt::format("PureMetadataObjectStorageOperation"); } std::string getInfoForLog() const override { return fmt::format("PureMetadataObjectStorageOperation"); }
}; };
struct ObjectsToRemove
{
StoredObjects objects;
UnlinkMetadataFileOperationOutcomePtr unlink_outcome;
};
struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation
{ {
std::string path; std::string path;
bool delete_metadata_only; bool delete_metadata_only;
StoredObjects objects_to_remove; ObjectsToRemove objects_to_remove;
bool if_exists; bool if_exists;
bool remove_from_cache = false; bool remove_from_cache = false;
@ -105,15 +112,12 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation
try try
{ {
uint32_t hardlink_count = metadata_storage.getHardlinkCount(path);
auto objects = metadata_storage.getStorageObjects(path); auto objects = metadata_storage.getStorageObjects(path);
tx->unlinkMetadata(path); auto unlink_outcome = tx->unlinkMetadata(path);
if (hardlink_count == 0) if (unlink_outcome)
{ objects_to_remove = ObjectsToRemove{std::move(objects), std::move(unlink_outcome)};
objects_to_remove = std::move(objects);
}
} }
catch (const Exception & e) catch (const Exception & e)
{ {
@ -142,8 +146,11 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation
/// due to network error or similar. And when it will retry an operation it may receive /// due to network error or similar. And when it will retry an operation it may receive
/// a 404 HTTP code. We don't want to threat this code as a real error for deletion process /// a 404 HTTP code. We don't want to threat this code as a real error for deletion process
/// (e.g. throwing some exceptions) and thus we just use method `removeObjectsIfExists` /// (e.g. throwing some exceptions) and thus we just use method `removeObjectsIfExists`
if (!delete_metadata_only && !objects_to_remove.empty()) if (!delete_metadata_only && !objects_to_remove.objects.empty()
object_storage.removeObjectsIfExist(objects_to_remove); && objects_to_remove.unlink_outcome->num_hardlinks == 0)
{
object_storage.removeObjectsIfExist(objects_to_remove.objects);
}
} }
}; };
@ -153,12 +160,6 @@ struct RemoveManyObjectStorageOperation final : public IDiskObjectStorageOperati
bool keep_all_batch_data; bool keep_all_batch_data;
NameSet file_names_remove_metadata_only; NameSet file_names_remove_metadata_only;
struct ObjectsToRemove
{
StoredObjects objects;
UnlinkMetadataFileOperationOutcomePtr unlink_outcome;
};
std::vector<ObjectsToRemove> objects_to_remove; std::vector<ObjectsToRemove> objects_to_remove;
bool remove_from_cache = false; bool remove_from_cache = false;
@ -197,10 +198,10 @@ struct RemoveManyObjectStorageOperation final : public IDiskObjectStorageOperati
try try
{ {
auto objects = metadata_storage.getStorageObjects(path);
auto unlink_outcome = tx->unlinkMetadata(path); auto unlink_outcome = tx->unlinkMetadata(path);
if (unlink_outcome && !keep_all_batch_data && !file_names_remove_metadata_only.contains(fs::path(path).filename())) if (unlink_outcome && !keep_all_batch_data && !file_names_remove_metadata_only.contains(fs::path(path).filename()))
{ {
auto objects = metadata_storage.getStorageObjects(path);
objects_to_remove.emplace_back(ObjectsToRemove{std::move(objects), std::move(unlink_outcome)}); objects_to_remove.emplace_back(ObjectsToRemove{std::move(objects), std::move(unlink_outcome)});
} }
} }
@ -244,10 +245,9 @@ struct RemoveManyObjectStorageOperation final : public IDiskObjectStorageOperati
struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOperation struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOperation
{ {
std::string path; std::string path;
std::unordered_map<std::string, StoredObjects> objects_to_remove; std::unordered_map<std::string, ObjectsToRemove> objects_to_remove_by_path;
bool keep_all_batch_data; bool keep_all_batch_data;
NameSet file_names_remove_metadata_only; NameSet file_names_remove_metadata_only;
StoredObjects objects_to_remove_from_cache;
RemoveRecursiveObjectStorageOperation( RemoveRecursiveObjectStorageOperation(
IObjectStorage & object_storage_, IObjectStorage & object_storage_,
@ -274,14 +274,11 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp
{ {
try try
{ {
uint32_t hardlink_count = metadata_storage.getHardlinkCount(path_to_remove);
auto objects_paths = metadata_storage.getStorageObjects(path_to_remove); auto objects_paths = metadata_storage.getStorageObjects(path_to_remove);
auto unlink_outcome = tx->unlinkMetadata(path_to_remove);
tx->unlinkMetadata(path_to_remove); if (unlink_outcome)
if (hardlink_count == 0)
{ {
objects_to_remove[path_to_remove] = std::move(objects_paths); objects_to_remove_by_path[path_to_remove] = ObjectsToRemove{std::move(objects_paths), std::move(unlink_outcome)};
} }
} }
catch (const Exception & e) catch (const Exception & e)
@ -331,11 +328,12 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp
if (!keep_all_batch_data) if (!keep_all_batch_data)
{ {
StoredObjects remove_from_remote; StoredObjects remove_from_remote;
for (auto && [local_path, remote_paths] : objects_to_remove) for (auto && [local_path, objects_to_remove] : objects_to_remove_by_path)
{ {
if (!file_names_remove_metadata_only.contains(fs::path(local_path).filename())) if (!file_names_remove_metadata_only.contains(fs::path(local_path).filename()))
{ {
std::move(remote_paths.begin(), remote_paths.end(), std::back_inserter(remove_from_remote)); if (objects_to_remove.unlink_outcome->num_hardlinks == 0)
std::move(objects_to_remove.objects.begin(), objects_to_remove.objects.end(), std::back_inserter(remove_from_remote));
} }
} }
/// Read comment inside RemoveObjectStorageOperation class /// Read comment inside RemoveObjectStorageOperation class