From 8ed4a5de622b06425e039464336b7f2a2b1dc6c3 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 17 May 2021 16:01:08 +0300 Subject: [PATCH] Fix Zero Copy after merge master --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 14 +++++++------- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 +- .../test_s3_zero_copy_replication/test.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index c0695f1e6e2..b9ece92e1fc 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1227,7 +1227,7 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_s3 "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: checksums.txt is missing", fullPath(disk, to)); /// If the part is not completely written, we cannot use fast path by listing files. - disk->removeRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(to + "/", keep_s3); } else { @@ -1240,17 +1240,17 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_s3 # pragma GCC diagnostic ignored "-Wunused-variable" #endif for (const auto & [file, _] : checksums.files) - disk->removeFile(to + "/" + file, keep_s3); + disk->removeSharedFile(to + "/" + file, keep_s3); #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeFile(to + "/" + file); - disk->removeFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); - disk->removeFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); + disk->removeSharedFile(to + "/" + file, keep_s3); + disk->removeSharedFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); + disk->removeSharedFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); - disk->removeDirectory(to); + disk->removeSharedRecursive(to, keep_s3); } catch (...) { @@ -1258,7 +1258,7 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_s3 LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(disk, to), getCurrentExceptionMessage(false)); - disk->removeRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(to + "/", keep_s3); } } } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index b13c4c5a700..50a6192935f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -136,7 +136,7 @@ public: void remove(bool keep_s3 = false) const; - void projectionRemove(const String & parent_to) const; + void projectionRemove(const String & parent_to, bool keep_s3 = false) const; /// Initialize columns (from columns.txt if exists, or create from column files if not). /// Load checksums from checksums.txt if exists. Load index if required. diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py index 731adb0da74..f1c52a2c234 100644 --- a/tests/integration/test_s3_zero_copy_replication/test.py +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -27,7 +27,7 @@ def cluster(): cluster.shutdown() -def get_large_objects_count(cluster, folder='data', size=100): +def get_large_objects_count(cluster, size=100, folder='data'): minio = cluster.minio_client counter = 0 for obj in minio.list_objects(cluster.minio_bucket, '{}/'.format(folder)): @@ -38,11 +38,11 @@ def get_large_objects_count(cluster, folder='data', size=100): def wait_for_large_objects_count(cluster, expected, size=100, timeout=30): while timeout > 0: - if get_large_objects_count(cluster, size) == expected: + if get_large_objects_count(cluster, size=size) == expected: return timeout -= 1 time.sleep(1) - assert get_large_objects_count(cluster, size) == expected + assert get_large_objects_count(cluster, size=size) == expected @pytest.mark.parametrize(