From f49936b9a067551e37069369c7cbbeca65f6807d Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Tue, 18 Oct 2022 17:53:42 +0200 Subject: [PATCH] remove tmp dir from inmemory part when freeze, more logs about old parts --- src/Compression/CachedCompressedReadBuffer.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeData.cpp | 17 +++++++++++++---- tests/queries/0_stateless/parts.lib | 8 ++++---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/Compression/CachedCompressedReadBuffer.cpp b/src/Compression/CachedCompressedReadBuffer.cpp index 8abc16ebb2a..47be6a9d328 100644 --- a/src/Compression/CachedCompressedReadBuffer.cpp +++ b/src/Compression/CachedCompressedReadBuffer.cpp @@ -47,8 +47,8 @@ bool CachedCompressedReadBuffer::nextImpl() auto cell = std::make_shared(); - size_t size_decompressed; - size_t size_compressed_without_checksum; + size_t size_decompressed = 0; + size_t size_compressed_without_checksum = 0; cell->compressed_size = readCompressedData(size_decompressed, size_compressed_without_checksum, false); if (cell->compressed_size) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 49cd01fc36b..0182c5fc904 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1721,7 +1721,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force) const auto time_now = std::chrono::system_clock::now(); auto now = std::chrono::system_clock::to_time_t(time_now); - const bool verbose_skip_part = (time_now - last_report_grab_old_parts) >= std::chrono::seconds(60); + const bool verbose_skip_part = (time_now - last_report_grab_old_parts) >= std::chrono::seconds(5); if (verbose_skip_part) last_report_grab_old_parts = time_now; @@ -6575,12 +6575,21 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( LOG_DEBUG(log, "Freezing part {} snapshot will be placed at {}", part->name, backup_path); auto data_part_storage = part->getDataPartStoragePtr(); - String src_part_path = data_part_storage->getRelativePath(); String backup_part_path = fs::path(backup_path) / relative_data_path; + + scope_guard src_flushed_tmp_dir_lock; + MergeTreeData::MutableDataPartPtr src_flushed_tmp_part; + if (auto part_in_memory = asInMemoryPart(part)) { - auto flushed_part_path = part_in_memory->getRelativePathForPrefix("tmp_freeze"); - data_part_storage = part_in_memory->flushToDisk(*flushed_part_path, metadata_snapshot); + auto flushed_part_path = *part_in_memory->getRelativePathForPrefix("tmp_freeze"); + src_flushed_tmp_dir_lock = part->storage.getTemporaryPartDirectoryHolder("tmp_freeze" + part->name); + + auto flushed_part_storage = part_in_memory->flushToDisk(flushed_part_path, metadata_snapshot); + src_flushed_tmp_part = createPart(part->name, part->info, flushed_part_storage); + src_flushed_tmp_part->is_temp = true; + + data_part_storage = flushed_part_storage; } auto callback = [this, &part, &backup_part_path](const DiskPtr & disk) diff --git a/tests/queries/0_stateless/parts.lib b/tests/queries/0_stateless/parts.lib index cd34e581307..f4b14de2e31 100644 --- a/tests/queries/0_stateless/parts.lib +++ b/tests/queries/0_stateless/parts.lib @@ -11,8 +11,8 @@ function wait_for_delete_empty_parts() res=$(${CLICKHOUSE_CLIENT} --query="SELECT count() FROM system.parts WHERE database='$database' AND table='$table' AND active AND rows=0") [[ $res -eq 0 ]] && return 0 - sleep 1 - timeout=$((timeout - 1)) + sleep 1.5 + timeout=$(echo "$timeout - 1.5" | bc) done echo "Timed out while waiting for delete empty parts!" >&2 @@ -30,8 +30,8 @@ function wait_for_delete_inactive_parts() res=$(${CLICKHOUSE_CLIENT} --query="SELECT count() FROM system.parts WHERE database='$database' AND table='$table' AND not active") [[ $res -eq 0 ]] && return 0 - sleep 1 - timeout=$((timeout - 1)) + sleep 1.5 + timeout=$(echo "$timeout - 1.5" | bc) done echo "Timed out while waiting for delete inactive parts!" >&2