mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-29 02:52:13 +00:00
946a126799
Buffer for reading from a compressed file with just checking checksums of the compressed blocks, without any decompression, so result can be proxied.
77 lines
2.5 KiB
C++
77 lines
2.5 KiB
C++
#include "CompressedReadBuffer.h"
|
|
#include <Compression/LZ4_decompress_faster.h>
|
|
|
|
|
|
namespace DB
|
|
{
|
|
|
|
bool CompressedReadBuffer::nextImpl()
|
|
{
|
|
size_t size_decompressed;
|
|
size_t size_compressed_without_checksum;
|
|
size_compressed = readCompressedData(size_decompressed, size_compressed_without_checksum, false);
|
|
if (!size_compressed)
|
|
return false;
|
|
|
|
auto additional_size_at_the_end_of_buffer = codec->getAdditionalSizeAtTheEndOfBuffer();
|
|
|
|
/// This is for clang static analyzer.
|
|
assert(size_decompressed + additional_size_at_the_end_of_buffer > 0);
|
|
|
|
memory.resize(size_decompressed + additional_size_at_the_end_of_buffer);
|
|
working_buffer = Buffer(memory.data(), &memory[size_decompressed]);
|
|
|
|
decompress(working_buffer.begin(), size_decompressed, size_compressed_without_checksum);
|
|
|
|
return true;
|
|
}
|
|
|
|
size_t CompressedReadBuffer::readBig(char * to, size_t n)
|
|
{
|
|
size_t bytes_read = 0;
|
|
|
|
/// If there are unread bytes in the buffer, then we copy necessary to `to`.
|
|
if (pos < working_buffer.end())
|
|
bytes_read += read(to, std::min(static_cast<size_t>(working_buffer.end() - pos), n));
|
|
|
|
/// If you need to read more - we will, if possible, uncompress at once to `to`.
|
|
while (bytes_read < n)
|
|
{
|
|
size_t size_decompressed;
|
|
size_t size_compressed_without_checksum;
|
|
|
|
if (!readCompressedData(size_decompressed, size_compressed_without_checksum, false))
|
|
return bytes_read;
|
|
|
|
auto additional_size_at_the_end_of_buffer = codec->getAdditionalSizeAtTheEndOfBuffer();
|
|
|
|
/// If the decompressed block fits entirely where it needs to be copied.
|
|
if (size_decompressed + additional_size_at_the_end_of_buffer <= n - bytes_read)
|
|
{
|
|
decompress(to + bytes_read, size_decompressed, size_compressed_without_checksum);
|
|
bytes_read += size_decompressed;
|
|
bytes += size_decompressed;
|
|
}
|
|
else
|
|
{
|
|
bytes += offset();
|
|
|
|
/// This is for clang static analyzer.
|
|
assert(size_decompressed + additional_size_at_the_end_of_buffer > 0);
|
|
|
|
memory.resize(size_decompressed + additional_size_at_the_end_of_buffer);
|
|
working_buffer = Buffer(memory.data(), &memory[size_decompressed]);
|
|
pos = working_buffer.begin();
|
|
|
|
decompress(working_buffer.begin(), size_decompressed, size_compressed_without_checksum);
|
|
|
|
bytes_read += read(to + bytes_read, n - bytes_read);
|
|
break;
|
|
}
|
|
}
|
|
|
|
return bytes_read;
|
|
}
|
|
|
|
}
|