From a419267dc6c5ad37ae30f4c7251db25f92311c61 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 8 Sep 2020 19:28:49 +0300 Subject: [PATCH] minor fixes --- src/Compression/CompressionCodecDelta.cpp | 1 - src/IO/ReadBuffer.h | 5 +++++ .../MergeTree/MergeTreeDataPartWriterCompact.cpp | 11 ++++++++++- src/Storages/MergeTree/checkDataPart.cpp | 4 ++-- .../01375_compact_parts_codecs.reference | 3 +++ .../0_stateless/01375_compact_parts_codecs.sql | 15 +++++++++++++++ 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp index dc866e527d6..ecb7c36b205 100644 --- a/src/Compression/CompressionCodecDelta.cpp +++ b/src/Compression/CompressionCodecDelta.cpp @@ -39,7 +39,6 @@ ASTPtr CompressionCodecDelta::getCodecDesc() const void CompressionCodecDelta::updateHash(SipHash & hash) const { getCodecDesc()->updateTreeHash(hash); - hash.update(delta_bytes_size); } namespace diff --git a/src/IO/ReadBuffer.h b/src/IO/ReadBuffer.h index a35e5206e49..3d6eb6970ce 100644 --- a/src/IO/ReadBuffer.h +++ b/src/IO/ReadBuffer.h @@ -123,6 +123,11 @@ public: return bytes_ignored; } + void ignoreAll() + { + tryIgnore(std::numeric_limits::max()); + } + /** Reads a single byte. */ bool ALWAYS_INLINE read(char & c) { diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index ab064689f47..9c3325d3d5a 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -112,8 +112,11 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) { auto & stream = compressed_streams[i]; + /// Offset should be 0, because compressed block is written for every granule. + assert(stream->hashing_buf.offset() == 0); + writeIntBinary(plain_hashing.count(), marks); - writeIntBinary(stream->hashing_buf.offset(), marks); + writeIntBinary(UInt64(0), marks); writeColumnSingleGranule(block.getByName(name_and_type->name), stream, current_row, rows_to_write); @@ -162,6 +165,12 @@ void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart: if (columns_buffer.size() != 0) writeBlock(header.cloneWithColumns(columns_buffer.releaseColumns())); +#ifndef NDEBUG + /// Offsets should be 0, because compressed block is written for every granule. + for (const auto & [_, stream] : streams_by_codec) + assert(stream->hashing_buf.offset() == 0); +#endif + if (with_final_mark && data_written) { for (size_t i = 0; i < columns_list.size(); ++i) diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 9d7e5315474..63b061b9702 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -89,7 +89,7 @@ IMergeTreeDataPart::Checksums checkDataPart( CompressedReadBuffer uncompressing_buf(compressed_hashing_buf); HashingReadBuffer uncompressed_hashing_buf(uncompressing_buf); - uncompressed_hashing_buf.tryIgnore(std::numeric_limits::max()); + uncompressed_hashing_buf.ignoreAll(); return IMergeTreeDataPart::Checksums::Checksum { compressed_hashing_buf.count(), compressed_hashing_buf.getHash(), @@ -102,7 +102,7 @@ IMergeTreeDataPart::Checksums checkDataPart( { auto file_buf = disk_->readFile(file_path); HashingReadBuffer hashing_buf(*file_buf); - hashing_buf.tryIgnore(std::numeric_limits::max()); + hashing_buf.ignoreAll(); return IMergeTreeDataPart::Checksums::Checksum{hashing_buf.count(), hashing_buf.getHash()}; }; diff --git a/tests/queries/0_stateless/01375_compact_parts_codecs.reference b/tests/queries/0_stateless/01375_compact_parts_codecs.reference index 24b3e22d9a6..b48892597b6 100644 --- a/tests/queries/0_stateless/01375_compact_parts_codecs.reference +++ b/tests/queries/0_stateless/01375_compact_parts_codecs.reference @@ -1,6 +1,9 @@ 12000 11890 499500 499500 999 +499500 499500 999 11965 11890 499500 499500 999 +499500 499500 999 5858 11890 499500 499500 999 +499500 499500 999 diff --git a/tests/queries/0_stateless/01375_compact_parts_codecs.sql b/tests/queries/0_stateless/01375_compact_parts_codecs.sql index 4b285f5bcc1..698f4148a15 100644 --- a/tests/queries/0_stateless/01375_compact_parts_codecs.sql +++ b/tests/queries/0_stateless/01375_compact_parts_codecs.sql @@ -10,6 +10,11 @@ SELECT sum(data_compressed_bytes), sum(data_uncompressed_bytes) SELECT sum(id), sum(val), max(s) FROM codecs; +DETACH TABLE codecs; +ATTACH table codecs; + +SELECT sum(id), sum(val), max(s) FROM codecs; + DROP TABLE codecs; CREATE TABLE codecs (id UInt32 CODEC(NONE), val UInt32 CODEC(NONE), s String CODEC(NONE)) @@ -22,6 +27,11 @@ SELECT sum(data_compressed_bytes), sum(data_uncompressed_bytes) SELECT sum(id), sum(val), max(s) FROM codecs; +DETACH TABLE codecs; +ATTACH table codecs; + +SELECT sum(id), sum(val), max(s) FROM codecs; + DROP TABLE codecs; CREATE TABLE codecs (id UInt32, val UInt32 CODEC(Delta, ZSTD), s String CODEC(ZSTD)) @@ -34,4 +44,9 @@ SELECT sum(data_compressed_bytes), sum(data_uncompressed_bytes) SELECT sum(id), sum(val), max(s) FROM codecs; +DETACH TABLE codecs; +ATTACH table codecs; + +SELECT sum(id), sum(val), max(s) FROM codecs; + DROP TABLE codecs;