thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 07:31:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Zero Copy after merge master

Browse Source
This commit is contained in:
Anton Ivashkin 2021-05-17 16:01:08 +03:00
parent 39a30b77fe
commit 8ed4a5de62
3 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/Storages/MergeTree/IMergeTreeDataPart.cpp
View File

@ -1227,7 +1227,7 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_s3
"Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: checksums.txt is missing", "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: checksums.txt is missing",
fullPath(disk, to)); fullPath(disk, to));
/// If the part is not completely written, we cannot use fast path by listing files. /// If the part is not completely written, we cannot use fast path by listing files.
disk->removeRecursive(to + "/", keep_s3); disk->removeSharedRecursive(to + "/", keep_s3);
} }
else else
{ {
@ -1240,17 +1240,17 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_s3
# pragma GCC diagnostic ignored "-Wunused-variable" # pragma GCC diagnostic ignored "-Wunused-variable"
#endif #endif
for (const auto & [file, _] : checksums.files) for (const auto & [file, _] : checksums.files)
disk->removeFile(to + "/" + file, keep_s3); disk->removeSharedFile(to + "/" + file, keep_s3);
#if !defined(__clang__) #if !defined(__clang__)
# pragma GCC diagnostic pop # pragma GCC diagnostic pop
#endif #endif
for (const auto & file : {"checksums.txt", "columns.txt"}) for (const auto & file : {"checksums.txt", "columns.txt"})
disk->removeFile(to + "/" + file); disk->removeSharedFile(to + "/" + file, keep_s3);
disk->removeFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); disk->removeSharedFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3);
disk->removeFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); disk->removeSharedFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3);
disk->removeDirectory(to); disk->removeSharedRecursive(to, keep_s3);
} }
catch (...) catch (...)
{ {
@ -1258,7 +1258,7 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_s3
LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(disk, to), getCurrentExceptionMessage(false)); LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(disk, to), getCurrentExceptionMessage(false));
disk->removeRecursive(to + "/", keep_s3); disk->removeSharedRecursive(to + "/", keep_s3);
} }
} }
} }

2
src/Storages/MergeTree/IMergeTreeDataPart.h
View File

@ -136,7 +136,7 @@ public:
void remove(bool keep_s3 = false) const; void remove(bool keep_s3 = false) const;
void projectionRemove(const String & parent_to) const; void projectionRemove(const String & parent_to, bool keep_s3 = false) const;
/// Initialize columns (from columns.txt if exists, or create from column files if not). /// Initialize columns (from columns.txt if exists, or create from column files if not).
/// Load checksums from checksums.txt if exists. Load index if required. /// Load checksums from checksums.txt if exists. Load index if required.

6
tests/integration/test_s3_zero_copy_replication/test.py
View File

@ -27,7 +27,7 @@ def cluster():
cluster.shutdown() cluster.shutdown()
def get_large_objects_count(cluster, folder='data', size=100): def get_large_objects_count(cluster, size=100, folder='data'):
minio = cluster.minio_client minio = cluster.minio_client
counter = 0 counter = 0
for obj in minio.list_objects(cluster.minio_bucket, '{}/'.format(folder)): for obj in minio.list_objects(cluster.minio_bucket, '{}/'.format(folder)):
@ -38,11 +38,11 @@ def get_large_objects_count(cluster, folder='data', size=100):
def wait_for_large_objects_count(cluster, expected, size=100, timeout=30): def wait_for_large_objects_count(cluster, expected, size=100, timeout=30):
while timeout > 0: while timeout > 0:
if get_large_objects_count(cluster, size) == expected: if get_large_objects_count(cluster, size=size) == expected:
return return
timeout -= 1 timeout -= 1
time.sleep(1) time.sleep(1)
assert get_large_objects_count(cluster, size) == expected assert get_large_objects_count(cluster, size=size) == expected
@pytest.mark.parametrize( @pytest.mark.parametrize(