diff --git a/contrib/xz b/contrib/xz new file mode 160000 index 00000000000..869b9d1b4ed --- /dev/null +++ b/contrib/xz @@ -0,0 +1 @@ +Subproject commit 869b9d1b4edd6df07f819d360d306251f8147353 diff --git a/src/IO/LzmaReadBuffer.cpp b/src/IO/LzmaReadBuffer.cpp index 7454df35a11..377b68a4ae3 100644 --- a/src/IO/LzmaReadBuffer.cpp +++ b/src/IO/LzmaReadBuffer.cpp @@ -6,11 +6,7 @@ namespace ErrorCodes { extern const int LZMA_STREAM_DECODER_FAILED; } -LzmaReadBuffer::LzmaReadBuffer( - std::unique_ptr in_, - size_t buf_size, - char * existing_memory, - size_t alignment) +LzmaReadBuffer::LzmaReadBuffer(std::unique_ptr in_, size_t buf_size, char * existing_memory, size_t alignment) : BufferWithOwnMemory(buf_size, existing_memory, alignment), in(std::move(in_)) { lstr = LZMA_STREAM_INIT; @@ -27,7 +23,8 @@ LzmaReadBuffer::LzmaReadBuffer( // lzma does not provide api for converting error code to string unlike zlib if (ret != LZMA_OK) throw Exception( - std::string("lzma_stream_decoder initialization failed: error code: ") + std::to_string(ret) + "; lzma version: " + LZMA_VERSION_STRING, + std::string("lzma_stream_decoder initialization failed: error code: ") + std::to_string(ret) + + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_DECODER_FAILED); } @@ -39,9 +36,8 @@ LzmaReadBuffer::~LzmaReadBuffer() bool LzmaReadBuffer::nextImpl() { if (eof) - { return false; - } + if (!lstr.avail_in) { @@ -63,16 +59,24 @@ bool LzmaReadBuffer::nextImpl() { eof = true; return working_buffer.size() != 0; - } else { - throw Exception(ErrorCodes::LZMA_STREAM_DECODER_FAILED, - "lzma decoder finished, but stream is still alive: error code: {}; lzma version: {}", ret, LZMA_VERSION_STRING); + } + else + { + throw Exception( + ErrorCodes::LZMA_STREAM_DECODER_FAILED, + "lzma decoder finished, but stream is still alive: error code: {}; lzma version: {}", + ret, + LZMA_VERSION_STRING); } } if (ret != LZMA_OK) - throw Exception(ErrorCodes::LZMA_STREAM_DECODER_FAILED, - "lzma_stream_decoder failed: error code: error codeL {}; lzma version: {}", ret, LZMA_VERSION_STRING); + throw Exception( + ErrorCodes::LZMA_STREAM_DECODER_FAILED, + "lzma_stream_decoder failed: error code: error codeL {}; lzma version: {}", + ret, + LZMA_VERSION_STRING); return true; } -} \ No newline at end of file +} diff --git a/src/IO/LzmaReadBuffer.h b/src/IO/LzmaReadBuffer.h index 3ece8a46b7d..ae4d4ccf749 100644 --- a/src/IO/LzmaReadBuffer.h +++ b/src/IO/LzmaReadBuffer.h @@ -30,5 +30,4 @@ private: lzma_stream lstr; bool eof; }; - } \ No newline at end of file diff --git a/src/IO/LzmaWriteBuffer.cpp b/src/IO/LzmaWriteBuffer.cpp index 59b50094334..395de00be1d 100644 --- a/src/IO/LzmaWriteBuffer.cpp +++ b/src/IO/LzmaWriteBuffer.cpp @@ -1,20 +1,16 @@ #include -namespace DB { +namespace DB +{ namespace ErrorCodes { extern const int LZMA_STREAM_ENCODER_FAILED; } LzmaWriteBuffer::LzmaWriteBuffer( - std::unique_ptr out_, - int compression_level, - size_t buf_size, - char * existing_memory, - size_t alignment) - : BufferWithOwnMemory(buf_size, existing_memory, alignment) - , out(std::move(out_)) + std::unique_ptr out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment) + : BufferWithOwnMemory(buf_size, existing_memory, alignment), out(std::move(out_)) { lstr = LZMA_STREAM_INIT; lstr.allocator = nullptr; @@ -22,23 +18,25 @@ LzmaWriteBuffer::LzmaWriteBuffer( lstr.avail_in = 0; lstr.next_out = nullptr; lstr.avail_out = 0; - + // options for further compression lzma_options_lzma opt_lzma2; if (lzma_lzma_preset(&opt_lzma2, compression_level)) - throw Exception(std::string("lzma preset failed: ") + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED); + throw Exception( + std::string("lzma preset failed: ") + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED); lzma_filter filters[] = { - { .id = LZMA_FILTER_X86, .options = NULL }, - { .id = LZMA_FILTER_LZMA2, .options = &opt_lzma2 }, - { .id = LZMA_VLI_UNKNOWN, .options = NULL }, + {.id = LZMA_FILTER_X86, .options = NULL}, + {.id = LZMA_FILTER_LZMA2, .options = &opt_lzma2}, + {.id = LZMA_VLI_UNKNOWN, .options = NULL}, }; lzma_ret ret = lzma_stream_encoder(&lstr, filters, LZMA_CHECK_CRC64); - if (ret != LZMA_OK) - throw Exception(std::string("lzma stream encoder init failed: ") + std::to_string(ret) + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED); - + if (ret != LZMA_OK) + throw Exception( + std::string("lzma stream encoder init failed: ") + std::to_string(ret) + "; lzma version: " + LZMA_VERSION_STRING, + ErrorCodes::LZMA_STREAM_ENCODER_FAILED); } LzmaWriteBuffer::~LzmaWriteBuffer() @@ -48,7 +46,8 @@ LzmaWriteBuffer::~LzmaWriteBuffer() finish(); lzma_end(&lstr); - } catch (...) + } + catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); } @@ -74,13 +73,14 @@ void LzmaWriteBuffer::nextImpl() out->position() = out->buffer().end() - lstr.avail_out; - if (ret == LZMA_STREAM_END) return; - + if (ret != LZMA_OK) - throw Exception(std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED); - + throw Exception( + std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING, + ErrorCodes::LZMA_STREAM_ENCODER_FAILED); + } while (lstr.avail_in > 0 || lstr.avail_out == 0); } @@ -102,16 +102,17 @@ void LzmaWriteBuffer::finish() out->position() = out->buffer().end() - lstr.avail_out; - if (ret == LZMA_STREAM_END) + if (ret == LZMA_STREAM_END) { finished = true; - return; + return; } - + if (ret != LZMA_OK) - throw Exception(std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED); - + throw Exception( + std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING, + ErrorCodes::LZMA_STREAM_ENCODER_FAILED); + } while (lstr.avail_out == 0); } - } diff --git a/src/IO/LzmaWriteBuffer.h b/src/IO/LzmaWriteBuffer.h index 986dc065c8c..fa5e3456f79 100644 --- a/src/IO/LzmaWriteBuffer.h +++ b/src/IO/LzmaWriteBuffer.h @@ -1,33 +1,32 @@ #pragma once -#include #include +#include #include -namespace DB { - +namespace DB +{ /// Performs compression using lzma library and writes compressed data to out_ WriteBuffer. class LzmaWriteBuffer : public BufferWithOwnMemory -{ +{ public: LzmaWriteBuffer( - std::unique_ptr out_, - int compression_level, - size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, - char * existing_memory = nullptr, - size_t alignment = 0); + std::unique_ptr out_, + int compression_level, + size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, + char * existing_memory = nullptr, + size_t alignment = 0); void finish(); ~LzmaWriteBuffer() override; private: -void nextImpl() override; + void nextImpl() override; -std::unique_ptr out; -lzma_stream lstr; -bool finished = false; + std::unique_ptr out; + lzma_stream lstr; + bool finished = false; }; - } \ No newline at end of file