mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-30 13:40:50 +00:00
98e3a99a88
Since this hides real problems, since destructor does final flush and if
it fails, then data will be lost.
One of such examples if MEMORY_LIMIT_EXCEEDED exception, so lock
exceptions from destructors, by using
MemoryTracker::LockExceptionInThread to block these exception, and allow
others (so std::terminate will be called, since this is c++11 with
noexcept for destructors by default).
Here is an example, that leads to empty block in the distributed batch:
2021.01.21 12:43:18.619739 [ 46468 ] {7bd60d75-ebcb-45d2-874d-260df9a4ddac} <Error> virtual DB::CompressedWriteBuffer::~CompressedWriteBuffer(): Code: 241, e.displayText() = DB::Exception: Memory limit (for user) exceeded: would use 332.07 GiB (attempt to allocate chunk of 4355342 bytes), maximum: 256.00 GiB, Stack trace (when copying this message, always include the lines below):
0. DB::Exception::Exception<>() @ 0x86f7b88 in /usr/bin/clickhouse
...
4. void DB::PODArrayBase<>::resize<>(unsigned long) @ 0xe9e878d in /usr/bin/clickhouse
5. DB::CompressedWriteBuffer::nextImpl() @ 0xe9f0296 in /usr/bin/clickhouse
6. DB::CompressedWriteBuffer::~CompressedWriteBuffer() @ 0xe9f0415 in /usr/bin/clickhouse
7. DB::DistributedBlockOutputStream::writeToShard() @ 0xf6bed4a in /usr/bin/clickhouse
132 lines
3.1 KiB
C++
132 lines
3.1 KiB
C++
#if !defined(ARCADIA_BUILD)
|
|
# include <Common/config.h>
|
|
#endif
|
|
|
|
#if USE_BROTLI
|
|
# include <IO/BrotliWriteBuffer.h>
|
|
# include <brotli/encode.h>
|
|
|
|
#include <Common/MemoryTracker.h>
|
|
|
|
namespace DB
|
|
{
|
|
|
|
namespace ErrorCodes
|
|
{
|
|
extern const int BROTLI_WRITE_FAILED;
|
|
}
|
|
|
|
|
|
class BrotliWriteBuffer::BrotliStateWrapper
|
|
{
|
|
public:
|
|
BrotliStateWrapper()
|
|
: state(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr))
|
|
{
|
|
}
|
|
|
|
~BrotliStateWrapper()
|
|
{
|
|
BrotliEncoderDestroyInstance(state);
|
|
}
|
|
|
|
BrotliEncoderState * state;
|
|
};
|
|
|
|
BrotliWriteBuffer::BrotliWriteBuffer(std::unique_ptr<WriteBuffer> out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment)
|
|
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment)
|
|
, brotli(std::make_unique<BrotliStateWrapper>())
|
|
, in_available(0)
|
|
, in_data(nullptr)
|
|
, out_capacity(0)
|
|
, out_data(nullptr)
|
|
, out(std::move(out_))
|
|
{
|
|
BrotliEncoderSetParameter(brotli->state, BROTLI_PARAM_QUALITY, static_cast<uint32_t>(compression_level));
|
|
// Set LZ77 window size. According to brotli sources default value is 24 (c/tools/brotli.c:81)
|
|
BrotliEncoderSetParameter(brotli->state, BROTLI_PARAM_LGWIN, 24);
|
|
}
|
|
|
|
BrotliWriteBuffer::~BrotliWriteBuffer()
|
|
{
|
|
/// FIXME move final flush into the caller
|
|
MemoryTracker::LockExceptionInThread lock;
|
|
finish();
|
|
}
|
|
|
|
void BrotliWriteBuffer::nextImpl()
|
|
{
|
|
if (!offset())
|
|
{
|
|
return;
|
|
}
|
|
|
|
in_data = reinterpret_cast<unsigned char *>(working_buffer.begin());
|
|
in_available = offset();
|
|
|
|
do
|
|
{
|
|
out->nextIfAtEnd();
|
|
out_data = reinterpret_cast<unsigned char *>(out->position());
|
|
out_capacity = out->buffer().end() - out->position();
|
|
|
|
int result = BrotliEncoderCompressStream(
|
|
brotli->state,
|
|
in_available ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
|
|
&in_available,
|
|
&in_data,
|
|
&out_capacity,
|
|
&out_data,
|
|
nullptr);
|
|
|
|
out->position() = out->buffer().end() - out_capacity;
|
|
|
|
if (result == 0)
|
|
{
|
|
throw Exception("brotli compress failed", ErrorCodes::BROTLI_WRITE_FAILED);
|
|
}
|
|
}
|
|
while (in_available > 0 || out_capacity == 0);
|
|
}
|
|
|
|
void BrotliWriteBuffer::finish()
|
|
{
|
|
if (finished)
|
|
return;
|
|
|
|
next();
|
|
|
|
while (true)
|
|
{
|
|
out->nextIfAtEnd();
|
|
out_data = reinterpret_cast<unsigned char *>(out->position());
|
|
out_capacity = out->buffer().end() - out->position();
|
|
|
|
int result = BrotliEncoderCompressStream(
|
|
brotli->state,
|
|
BROTLI_OPERATION_FINISH,
|
|
&in_available,
|
|
&in_data,
|
|
&out_capacity,
|
|
&out_data,
|
|
nullptr);
|
|
|
|
out->position() = out->buffer().end() - out_capacity;
|
|
|
|
if (BrotliEncoderIsFinished(brotli->state))
|
|
{
|
|
finished = true;
|
|
return;
|
|
}
|
|
|
|
if (result == 0)
|
|
{
|
|
throw Exception("brotli compress failed", ErrorCodes::BROTLI_WRITE_FAILED);
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|