From 9ec59cff3c64a0a8661003dfa8e31d8f90c32ae1 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Mon, 9 Sep 2024 10:08:54 +0000 Subject: [PATCH] Disable checksums for new file in parts --- src/Storages/MergeTree/DataPartsExchange.cpp | 3 ++- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 3 +++ .../MergeTree/MergedBlockOutputStream.cpp | 24 ++++++++----------- ...ms_in_system_parts_columns_table.reference | 2 +- .../02253_empty_part_checksums.reference | 2 +- .../0_stateless/02361_fsync_profile_events.sh | 2 +- ...1_compress_marks_and_primary_key.reference | 8 +++---- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 061ee356203..a9990d38652 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -777,7 +777,8 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( if (file_name != "checksums.txt" && file_name != "columns.txt" && file_name != IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME && - file_name != IMergeTreeDataPart::METADATA_VERSION_FILE_NAME) + file_name != IMergeTreeDataPart::METADATA_VERSION_FILE_NAME && + file_name != IMergeTreeDataPart::MIN_MAX_TIME_OF_DATA_INSERT_FILE) checksums.addFile(file_name, file_size, expected_hash); } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index c30d80f2379..7fb18ce1ba7 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1015,6 +1015,9 @@ NameSet IMergeTreeDataPart::getFileNamesWithoutChecksums() const if (getDataPartStorage().exists(METADATA_VERSION_FILE_NAME)) result.emplace(METADATA_VERSION_FILE_NAME); + if (getDataPartStorage().exists(MIN_MAX_TIME_OF_DATA_INSERT_FILE)) + result.emplace(MIN_MAX_TIME_OF_DATA_INSERT_FILE); + return result; } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 3a00c5cf431..1e16ffb7fb1 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -315,20 +315,6 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis written_files.emplace_back(std::move(out)); } - { - auto out = new_part->getDataPartStorage().writeFile(IMergeTreeDataPart::MIN_MAX_TIME_OF_DATA_INSERT_FILE, 4096, write_settings); - HashingWriteBuffer out_hashing(*out); - DB::writeIntText(*new_part->min_time_of_data_insert, out_hashing); - DB::writeText(" ", out_hashing); - DB::writeIntText(*new_part->max_time_of_data_insert, out_hashing); - out_hashing.finalize(); - checksums.files[IMergeTreeDataPart::MIN_MAX_TIME_OF_DATA_INSERT_FILE].file_size = out_hashing.count(); - checksums.files[IMergeTreeDataPart::MIN_MAX_TIME_OF_DATA_INSERT_FILE].file_hash = out_hashing.getHash(); - - out->preFinalize(); - written_files.emplace_back(std::move(out)); - } - { /// Write a file with a description of columns. auto out = new_part->getDataPartStorage().writeFile("columns.txt", 4096, write_settings); @@ -345,6 +331,16 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis written_files.emplace_back(std::move(out)); } + { + auto out = new_part->getDataPartStorage().writeFile(IMergeTreeDataPart::MIN_MAX_TIME_OF_DATA_INSERT_FILE, 4096, write_settings); + DB::writeIntText(*new_part->min_time_of_data_insert, *out); + DB::writeText(" ", *out); + DB::writeIntText(*new_part->max_time_of_data_insert, *out); + + out->preFinalize(); + written_files.emplace_back(std::move(out)); + } + if (default_codec != nullptr) { auto out = new_part->getDataPartStorage().writeFile(IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096, write_settings); diff --git a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference index 4a896a46fed..4bf3cfe65a2 100644 --- a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference +++ b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference @@ -1 +1 @@ -20000101_1_1_0 test_00961 812036551f93a3685116fa6169d36fa9 2ca3d19eefed0ce7b4e7627e443b9a59 908ddf2b1d0af239da96ff1e527a8a1f +20000101_1_1_0 test_00961 e4ed027389c208d2b5fce9c4ef1ca42c 4c23d7f5920f89aefc3b062b646cd23d 908ddf2b1d0af239da96ff1e527a8a1f diff --git a/tests/queries/0_stateless/02253_empty_part_checksums.reference b/tests/queries/0_stateless/02253_empty_part_checksums.reference index 0f2374adab9..65a8c9ee65e 100644 --- a/tests/queries/0_stateless/02253_empty_part_checksums.reference +++ b/tests/queries/0_stateless/02253_empty_part_checksums.reference @@ -5,4 +5,4 @@ 0 1 0 -0_0_0_0 Wide 3f2fc7b294c1676dcdc3a7c830228aa9 645e5774e2921598cd105516a04c6a8b b324ada5cd6bb14402c1e59200bd003a +0_0_0_0 Wide 370db59d5dcaef5d762b11d319c368c7 514a8be2dac94fd039dbd230065e58a4 b324ada5cd6bb14402c1e59200bd003a diff --git a/tests/queries/0_stateless/02361_fsync_profile_events.sh b/tests/queries/0_stateless/02361_fsync_profile_events.sh index 8d8c69e32a5..d89f08da168 100755 --- a/tests/queries/0_stateless/02361_fsync_profile_events.sh +++ b/tests/queries/0_stateless/02361_fsync_profile_events.sh @@ -45,7 +45,7 @@ for i in {1..100}; do ")" # Non retriable errors - if [[ $FileSync -ne 8 ]]; then + if [[ $FileSync -ne 9 ]]; then echo "FileSync: $FileSync != 9" >&2 exit 2 fi diff --git a/tests/queries/0_stateless/02381_compress_marks_and_primary_key.reference b/tests/queries/0_stateless/02381_compress_marks_and_primary_key.reference index ed7af3b2f4f..53bddb77b84 100644 --- a/tests/queries/0_stateless/02381_compress_marks_and_primary_key.reference +++ b/tests/queries/0_stateless/02381_compress_marks_and_primary_key.reference @@ -1,13 +1,13 @@ 1000 10000 1000 10000 -test_02381 2000000 16112832 11904 16100928 -test_02381_compress 2000000 16099668 1658 16098010 +test_02381 2000000 16112790 11904 16100886 +test_02381_compress 2000000 16099626 1658 16097968 10000 100000 10000 100000 10000 100000 10000 100000 -test_02381 4000000 28098376 2946 28095430 -test_02381_compress 4000000 28125454 23616 28101838 +test_02381 4000000 28098334 2946 28095388 +test_02381_compress 4000000 28125412 23616 28101796 1 Hello 2 World 1 Hello