thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

compressor: added --hc mode [#METR-2944].

Browse Source
This commit is contained in:
Alexey Milovidov 2014-03-02 09:12:51 +04:00
parent e51227f502
commit 713d0c468e
3 changed files with 21 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dbms/include/DB/IO/CompressedStream.h
View File

@ -18,6 +18,7 @@ namespace CompressionMethod
{
QuickLZ = 0,
LZ4 = 1,
LZ4HC = 2, /// Формат такой же, как у LZ4. Разница только при сжатии.
};
}

16
dbms/include/DB/IO/CompressedWriteBuffer.h
View File

@ -7,6 +7,7 @@
#include <city.h>
#include <quicklz/quicklz_level1.h>
#include <lz4/lz4.h>
#include <lz4/lz4hc.h>
#include <DB/Common/PODArray.h>
#include <DB/Core/Types.h>
@ -58,6 +59,7 @@ private:
break;
}
case CompressionMethod::LZ4:
case CompressionMethod::LZ4HC:
{
/** В случае LZ4, в начале запишем заголовок такого же размера и структуры, как в QuickLZ
* 1 байт, чтобы отличить LZ4 от QuickLZ.
@ -68,10 +70,16 @@ private:
compressed_buffer[0] = 0x82; /// Второй бит - для совместимости с QuickLZ - обозначает, что размеры записываются 4 байтами.
compressed_size = QUICKLZ_HEADER_SIZE + LZ4_compress(
working_buffer.begin(),
&compressed_buffer[QUICKLZ_HEADER_SIZE],
uncompressed_size);
if (method == CompressionMethod::LZ4)
compressed_size = QUICKLZ_HEADER_SIZE + LZ4_compress(
working_buffer.begin(),
&compressed_buffer[QUICKLZ_HEADER_SIZE],
uncompressed_size);
else
compressed_size = QUICKLZ_HEADER_SIZE + LZ4_compressHC(
working_buffer.begin(),
&compressed_buffer[QUICKLZ_HEADER_SIZE],
uncompressed_size);
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;

9
utils/compressor/main.cpp
View File

@ -43,6 +43,7 @@ int main(int argc, char ** argv)
("help,h", "produce help message")
("d,decompress", "decompress")
("block-size,b", boost::program_options::value<unsigned>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size")
("hc", "use LZ4HC instead of LZ4")
("qlz", "use QuickLZ (level 1) instead of LZ4")
("stat", "print block statistics of compressed data")
;
@ -61,10 +62,16 @@ int main(int argc, char ** argv)
{
bool decompress = options.count("d");
bool use_qlz = options.count("qlz");;
bool use_lz4hc = options.count("hc");;
bool stat_mode = options.count("stat");
unsigned block_size = options["block-size"].as<unsigned>();
DB::CompressionMethod::Enum method = use_qlz ? DB::CompressionMethod::QuickLZ : DB::CompressionMethod::LZ4;
DB::CompressionMethod::Enum method = DB::CompressionMethod::LZ4;
if (use_qlz)
method = DB::CompressionMethod::QuickLZ;
else if (use_lz4hc)
method = DB::CompressionMethod::LZ4HC;
DB::ReadBufferFromFileDescriptor rb(STDIN_FILENO);
DB::WriteBufferFromFileDescriptor wb(STDOUT_FILENO);