From f2ff785460ba5876fd608680a374861498c0bb91 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Jan 2021 02:18:57 +0300 Subject: [PATCH 1/2] Do not check bit flips for big buffers (since the size can be corrupted) --- src/Compression/CompressedReadBufferBase.cpp | 27 ++++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index 8b16b68a999..74c902ce492 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -67,20 +67,25 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c buf[pos / 8] ^= 1 << pos % 8; }; - /// Check if the difference caused by single bit flip in data. - for (size_t bit_pos = 0; bit_pos < size * 8; ++bit_pos) + /// If size is too huge, then this may be caused by corruption. + /// And anyway this is pretty heavy, so avoid burning too much CPU here. + if (size < 1<<20) { - flip_bit(data, bit_pos); - - auto checksum_of_data_with_flipped_bit = CityHash_v1_0_2::CityHash128(data, size); - if (expected_checksum == checksum_of_data_with_flipped_bit) + /// Check if the difference caused by single bit flip in data. + for (size_t bit_pos = 0; bit_pos < size * 8; ++bit_pos) { - message << ". The mismatch is caused by single bit flip in data block at byte " << (bit_pos / 8) << ", bit " << (bit_pos % 8) << ". " - << message_hardware_failure; - throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); - } + flip_bit(data, bit_pos); - flip_bit(data, bit_pos); /// Restore + auto checksum_of_data_with_flipped_bit = CityHash_v1_0_2::CityHash128(data, size); + if (expected_checksum == checksum_of_data_with_flipped_bit) + { + message << ". The mismatch is caused by single bit flip in data block at byte " << (bit_pos / 8) << ", bit " << (bit_pos % 8) << ". " + << message_hardware_failure; + throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + } + + flip_bit(data, bit_pos); /// Restore + } } /// Check if the difference caused by single bit flip in stored checksum. From 6be8338584dacbf6d0f834256ce06a246289cb8c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 8 Jan 2021 22:41:24 +0300 Subject: [PATCH 2/2] Update CompressedReadBufferBase.cpp --- src/Compression/CompressedReadBufferBase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index 74c902ce492..5be31913140 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -69,7 +69,7 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c /// If size is too huge, then this may be caused by corruption. /// And anyway this is pretty heavy, so avoid burning too much CPU here. - if (size < 1<<20) + if (size < (1ULL << 20)) { /// Check if the difference caused by single bit flip in data. for (size_t bit_pos = 0; bit_pos < size * 8; ++bit_pos)