fixed whitespaces, added hidden submodule file

This commit is contained in:
a.palagashvili 2020-11-02 23:04:49 +03:00
parent 2ad01c59da
commit ba6fa5d828
5 changed files with 60 additions and 56 deletions

1
contrib/xz vendored Submodule

@ -0,0 +1 @@
Subproject commit 869b9d1b4edd6df07f819d360d306251f8147353

View File

@ -6,11 +6,7 @@ namespace ErrorCodes
{ {
extern const int LZMA_STREAM_DECODER_FAILED; extern const int LZMA_STREAM_DECODER_FAILED;
} }
LzmaReadBuffer::LzmaReadBuffer( LzmaReadBuffer::LzmaReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char * existing_memory, size_t alignment)
std::unique_ptr<ReadBuffer> in_,
size_t buf_size,
char * existing_memory,
size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment), in(std::move(in_)) : BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment), in(std::move(in_))
{ {
lstr = LZMA_STREAM_INIT; lstr = LZMA_STREAM_INIT;
@ -27,7 +23,8 @@ LzmaReadBuffer::LzmaReadBuffer(
// lzma does not provide api for converting error code to string unlike zlib // lzma does not provide api for converting error code to string unlike zlib
if (ret != LZMA_OK) if (ret != LZMA_OK)
throw Exception( throw Exception(
std::string("lzma_stream_decoder initialization failed: error code: ") + std::to_string(ret) + "; lzma version: " + LZMA_VERSION_STRING, std::string("lzma_stream_decoder initialization failed: error code: ") + std::to_string(ret)
+ "; lzma version: " + LZMA_VERSION_STRING,
ErrorCodes::LZMA_STREAM_DECODER_FAILED); ErrorCodes::LZMA_STREAM_DECODER_FAILED);
} }
@ -39,9 +36,8 @@ LzmaReadBuffer::~LzmaReadBuffer()
bool LzmaReadBuffer::nextImpl() bool LzmaReadBuffer::nextImpl()
{ {
if (eof) if (eof)
{
return false; return false;
}
if (!lstr.avail_in) if (!lstr.avail_in)
{ {
@ -63,16 +59,24 @@ bool LzmaReadBuffer::nextImpl()
{ {
eof = true; eof = true;
return working_buffer.size() != 0; return working_buffer.size() != 0;
} else { }
throw Exception(ErrorCodes::LZMA_STREAM_DECODER_FAILED, else
"lzma decoder finished, but stream is still alive: error code: {}; lzma version: {}", ret, LZMA_VERSION_STRING); {
throw Exception(
ErrorCodes::LZMA_STREAM_DECODER_FAILED,
"lzma decoder finished, but stream is still alive: error code: {}; lzma version: {}",
ret,
LZMA_VERSION_STRING);
} }
} }
if (ret != LZMA_OK) if (ret != LZMA_OK)
throw Exception(ErrorCodes::LZMA_STREAM_DECODER_FAILED, throw Exception(
"lzma_stream_decoder failed: error code: error codeL {}; lzma version: {}", ret, LZMA_VERSION_STRING); ErrorCodes::LZMA_STREAM_DECODER_FAILED,
"lzma_stream_decoder failed: error code: error codeL {}; lzma version: {}",
ret,
LZMA_VERSION_STRING);
return true; return true;
} }
} }

View File

@ -30,5 +30,4 @@ private:
lzma_stream lstr; lzma_stream lstr;
bool eof; bool eof;
}; };
} }

View File

@ -1,20 +1,16 @@
#include <IO/LzmaWriteBuffer.h> #include <IO/LzmaWriteBuffer.h>
namespace DB { namespace DB
{
namespace ErrorCodes namespace ErrorCodes
{ {
extern const int LZMA_STREAM_ENCODER_FAILED; extern const int LZMA_STREAM_ENCODER_FAILED;
} }
LzmaWriteBuffer::LzmaWriteBuffer( LzmaWriteBuffer::LzmaWriteBuffer(
std::unique_ptr<WriteBuffer> out_, std::unique_ptr<WriteBuffer> out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment)
int compression_level, : BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment), out(std::move(out_))
size_t buf_size,
char * existing_memory,
size_t alignment)
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment)
, out(std::move(out_))
{ {
lstr = LZMA_STREAM_INIT; lstr = LZMA_STREAM_INIT;
lstr.allocator = nullptr; lstr.allocator = nullptr;
@ -22,23 +18,25 @@ LzmaWriteBuffer::LzmaWriteBuffer(
lstr.avail_in = 0; lstr.avail_in = 0;
lstr.next_out = nullptr; lstr.next_out = nullptr;
lstr.avail_out = 0; lstr.avail_out = 0;
// options for further compression // options for further compression
lzma_options_lzma opt_lzma2; lzma_options_lzma opt_lzma2;
if (lzma_lzma_preset(&opt_lzma2, compression_level)) if (lzma_lzma_preset(&opt_lzma2, compression_level))
throw Exception(std::string("lzma preset failed: ") + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED); throw Exception(
std::string("lzma preset failed: ") + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED);
lzma_filter filters[] = { lzma_filter filters[] = {
{ .id = LZMA_FILTER_X86, .options = NULL }, {.id = LZMA_FILTER_X86, .options = NULL},
{ .id = LZMA_FILTER_LZMA2, .options = &opt_lzma2 }, {.id = LZMA_FILTER_LZMA2, .options = &opt_lzma2},
{ .id = LZMA_VLI_UNKNOWN, .options = NULL }, {.id = LZMA_VLI_UNKNOWN, .options = NULL},
}; };
lzma_ret ret = lzma_stream_encoder(&lstr, filters, LZMA_CHECK_CRC64); lzma_ret ret = lzma_stream_encoder(&lstr, filters, LZMA_CHECK_CRC64);
if (ret != LZMA_OK) if (ret != LZMA_OK)
throw Exception(std::string("lzma stream encoder init failed: ") + std::to_string(ret) + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED); throw Exception(
std::string("lzma stream encoder init failed: ") + std::to_string(ret) + "; lzma version: " + LZMA_VERSION_STRING,
ErrorCodes::LZMA_STREAM_ENCODER_FAILED);
} }
LzmaWriteBuffer::~LzmaWriteBuffer() LzmaWriteBuffer::~LzmaWriteBuffer()
@ -48,7 +46,8 @@ LzmaWriteBuffer::~LzmaWriteBuffer()
finish(); finish();
lzma_end(&lstr); lzma_end(&lstr);
} catch (...) }
catch (...)
{ {
tryLogCurrentException(__PRETTY_FUNCTION__); tryLogCurrentException(__PRETTY_FUNCTION__);
} }
@ -74,13 +73,14 @@ void LzmaWriteBuffer::nextImpl()
out->position() = out->buffer().end() - lstr.avail_out; out->position() = out->buffer().end() - lstr.avail_out;
if (ret == LZMA_STREAM_END) if (ret == LZMA_STREAM_END)
return; return;
if (ret != LZMA_OK) if (ret != LZMA_OK)
throw Exception(std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED); throw Exception(
std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING,
ErrorCodes::LZMA_STREAM_ENCODER_FAILED);
} while (lstr.avail_in > 0 || lstr.avail_out == 0); } while (lstr.avail_in > 0 || lstr.avail_out == 0);
} }
@ -102,16 +102,17 @@ void LzmaWriteBuffer::finish()
out->position() = out->buffer().end() - lstr.avail_out; out->position() = out->buffer().end() - lstr.avail_out;
if (ret == LZMA_STREAM_END) if (ret == LZMA_STREAM_END)
{ {
finished = true; finished = true;
return; return;
} }
if (ret != LZMA_OK) if (ret != LZMA_OK)
throw Exception(std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING, ErrorCodes::LZMA_STREAM_ENCODER_FAILED); throw Exception(
std::string("lzma stream encoding failed: ") + "; lzma version: " + LZMA_VERSION_STRING,
ErrorCodes::LZMA_STREAM_ENCODER_FAILED);
} while (lstr.avail_out == 0); } while (lstr.avail_out == 0);
} }
} }

View File

@ -1,33 +1,32 @@
#pragma once #pragma once
#include <IO/WriteBuffer.h>
#include <IO/BufferWithOwnMemory.h> #include <IO/BufferWithOwnMemory.h>
#include <IO/WriteBuffer.h>
#include <lzma.h> #include <lzma.h>
namespace DB { namespace DB
{
/// Performs compression using lzma library and writes compressed data to out_ WriteBuffer. /// Performs compression using lzma library and writes compressed data to out_ WriteBuffer.
class LzmaWriteBuffer : public BufferWithOwnMemory<WriteBuffer> class LzmaWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
{ {
public: public:
LzmaWriteBuffer( LzmaWriteBuffer(
std::unique_ptr<WriteBuffer> out_, std::unique_ptr<WriteBuffer> out_,
int compression_level, int compression_level,
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
char * existing_memory = nullptr, char * existing_memory = nullptr,
size_t alignment = 0); size_t alignment = 0);
void finish(); void finish();
~LzmaWriteBuffer() override; ~LzmaWriteBuffer() override;
private: private:
void nextImpl() override; void nextImpl() override;
std::unique_ptr<WriteBuffer> out; std::unique_ptr<WriteBuffer> out;
lzma_stream lstr; lzma_stream lstr;
bool finished = false; bool finished = false;
}; };
} }