ClickHouse/src/IO/BrotliWriteBuffer.cpp

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

130 lines
3.1 KiB
C++
Raw Normal View History

#include "config.h"
#if USE_BROTLI
# include <IO/BrotliWriteBuffer.h>
# include <brotli/encode.h>
namespace DB
{
2019-04-10 16:12:31 +00:00
namespace ErrorCodes
{
extern const int BROTLI_WRITE_FAILED;
}
class BrotliWriteBuffer::BrotliStateWrapper
{
public:
BrotliStateWrapper()
: state(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr))
{
}
~BrotliStateWrapper()
{
BrotliEncoderDestroyInstance(state);
}
BrotliEncoderState * state;
};
BrotliWriteBuffer::BrotliWriteBuffer(std::unique_ptr<WriteBuffer> out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment)
2021-11-10 22:58:56 +00:00
: WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment)
, brotli(std::make_unique<BrotliStateWrapper>())
, in_available(0)
, in_data(nullptr)
, out_capacity(0)
, out_data(nullptr)
{
BrotliEncoderSetParameter(brotli->state, BROTLI_PARAM_QUALITY, static_cast<uint32_t>(compression_level));
// Set LZ77 window size. According to brotli sources default value is 24 (c/tools/brotli.c:81)
BrotliEncoderSetParameter(brotli->state, BROTLI_PARAM_LGWIN, 24);
}
BrotliWriteBuffer::~BrotliWriteBuffer()
{
2021-11-11 17:27:23 +00:00
finalize();
2019-03-19 20:18:41 +00:00
}
2019-03-19 20:18:41 +00:00
void BrotliWriteBuffer::nextImpl()
{
if (!offset())
{
return;
}
2019-03-19 20:18:41 +00:00
in_data = reinterpret_cast<unsigned char *>(working_buffer.begin());
in_available = offset();
Fix abnormal server termination for nested writers Writers with nested writer can call next() from the dtor for nested writer and this will cause exception again, so the buffer position should be updated on exceptions. Found by stress test (thread) here [1] and here [2]: 2021.02.27 19:27:53.498977 [ 302 ] {} <Fatal> BaseDaemon: (version 21.3.1.6130, build id: 2DAEC5DEBF03C5A1C3BF66B7779C886F16239345) (from thread 1614) Terminate called for uncaught exception: Code: 24, e.displayText() = DB::Exception: Cannot write to ostream at offset 2097498, Stack trace (when copying this message, always include the lines below): 0. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/exception:0: Poco::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) @ 0x15bef2ab in /usr/bin/clickhouse 1. ./obj-x86_64-linux-gnu/../src/Common/Exception.cpp:56: DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0x8aea92e in /usr/bin/clickhouse 2. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:0: DB::WriteBufferFromOStream::nextImpl() @ 0x8bbbc45 in /usr/bin/clickhouse 3. ./obj-x86_64-linux-gnu/../src/IO/BufferBase.h:39: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8bbc077 in /usr/bin/clickhouse 4. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:44: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8bbc18a in /usr/bin/clickhouse 5. ./obj-x86_64-linux-gnu/../src/IO/BufferWithOwnMemory.h:137: DB::ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer() @ 0x118bdc29 in /usr/bin/clickhouse 6. ./obj-x86_64-linux-gnu/../src/IO/ZstdDeflatingWriteBuffer.cpp:32: DB::ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer() @ 0x118be3ea in /usr/bin/clickhouse 7. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:0: DB::WriteBufferFromHTTPServerResponse::finalize() @ 0x12f1dceb in /usr/bin/clickhouse 8. ./obj-x86_64-linux-gnu/../src/Server/HTTPHandler.cpp:703: DB::HTTPHandler::trySendExceptionToClient(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, DB::HTTPServerRequest&, DB::HTTPServerResponse&, DB::HTTPHandler::Output&) @ 0x12e9fecc in /usr/bin/clickhouse 9. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/string:1444: DB::HTTPHandler::handleRequest(DB::HTTPServerRequest&, DB::HTTPServerResponse&) @ 0x12ea0d60 in /usr/bin/clickhouse 10. ./obj-x86_64-linux-gnu/../src/Server/HTTP/HTTPServerConnection.cpp:0: DB::HTTPServerConnection::run() @ 0x12f16db1 in /usr/bin/clickhouse 11. ./obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerConnection.cpp:57: Poco::Net::TCPServerConnection::start() @ 0x15b184f3 in /usr/bin/clickhouse 12. ./obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerDispatcher.cpp:0: Poco::Net::TCPServerDispatcher::run() @ 0x15b18c1f in /usr/bin/clickhouse 13. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/ScopedLock.h:36: Poco::PooledThread::run() @ 0x15c7fdb2 in /usr/bin/clickhouse 14. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread.cpp:56: Poco::(anonymous namespace)::RunnableHolder::run() @ 0x15c7e350 in /usr/bin/clickhouse 15. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/SharedPtr.h:277: Poco::ThreadImpl::runnableEntry(void*) @ 0x15c7cb58 in /usr/bin/clickhouse 16. __tsan_thread_start_func @ 0x8a04ced in /usr/bin/clickhouse 17. start_thread @ 0x9609 in /usr/lib/x86_64-linux-gnu/libpthread-2.31.so 18. __clone @ 0x122293 in /usr/lib/x86_64-linux-gnu/libc-2.31.so (version 21.3.1.6130) [1]: https://clickhouse-test-reports.s3.yandex.net/21279/4f61ef3099f42f17b496a0b0424773978d9a32dc/stress_test_(thread).html#fail1 [2]: https://clickhouse-test-reports.s3.yandex.net/21292/ae9fea1d0af118a8f87b224d194d61da1567188b/stress_test_(thread).html#fail1 v2: https://clickhouse-test-reports.s3.yandex.net/21305/e969daa6e86c5e09cfef08cfde19712982b64e59/stress_test_(thread).html#fail1
2021-02-28 07:18:49 +00:00
try
{
Fix abnormal server termination for nested writers Writers with nested writer can call next() from the dtor for nested writer and this will cause exception again, so the buffer position should be updated on exceptions. Found by stress test (thread) here [1] and here [2]: 2021.02.27 19:27:53.498977 [ 302 ] {} <Fatal> BaseDaemon: (version 21.3.1.6130, build id: 2DAEC5DEBF03C5A1C3BF66B7779C886F16239345) (from thread 1614) Terminate called for uncaught exception: Code: 24, e.displayText() = DB::Exception: Cannot write to ostream at offset 2097498, Stack trace (when copying this message, always include the lines below): 0. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/exception:0: Poco::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) @ 0x15bef2ab in /usr/bin/clickhouse 1. ./obj-x86_64-linux-gnu/../src/Common/Exception.cpp:56: DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0x8aea92e in /usr/bin/clickhouse 2. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:0: DB::WriteBufferFromOStream::nextImpl() @ 0x8bbbc45 in /usr/bin/clickhouse 3. ./obj-x86_64-linux-gnu/../src/IO/BufferBase.h:39: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8bbc077 in /usr/bin/clickhouse 4. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:44: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8bbc18a in /usr/bin/clickhouse 5. ./obj-x86_64-linux-gnu/../src/IO/BufferWithOwnMemory.h:137: DB::ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer() @ 0x118bdc29 in /usr/bin/clickhouse 6. ./obj-x86_64-linux-gnu/../src/IO/ZstdDeflatingWriteBuffer.cpp:32: DB::ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer() @ 0x118be3ea in /usr/bin/clickhouse 7. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:0: DB::WriteBufferFromHTTPServerResponse::finalize() @ 0x12f1dceb in /usr/bin/clickhouse 8. ./obj-x86_64-linux-gnu/../src/Server/HTTPHandler.cpp:703: DB::HTTPHandler::trySendExceptionToClient(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, DB::HTTPServerRequest&, DB::HTTPServerResponse&, DB::HTTPHandler::Output&) @ 0x12e9fecc in /usr/bin/clickhouse 9. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/string:1444: DB::HTTPHandler::handleRequest(DB::HTTPServerRequest&, DB::HTTPServerResponse&) @ 0x12ea0d60 in /usr/bin/clickhouse 10. ./obj-x86_64-linux-gnu/../src/Server/HTTP/HTTPServerConnection.cpp:0: DB::HTTPServerConnection::run() @ 0x12f16db1 in /usr/bin/clickhouse 11. ./obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerConnection.cpp:57: Poco::Net::TCPServerConnection::start() @ 0x15b184f3 in /usr/bin/clickhouse 12. ./obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerDispatcher.cpp:0: Poco::Net::TCPServerDispatcher::run() @ 0x15b18c1f in /usr/bin/clickhouse 13. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/ScopedLock.h:36: Poco::PooledThread::run() @ 0x15c7fdb2 in /usr/bin/clickhouse 14. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread.cpp:56: Poco::(anonymous namespace)::RunnableHolder::run() @ 0x15c7e350 in /usr/bin/clickhouse 15. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/SharedPtr.h:277: Poco::ThreadImpl::runnableEntry(void*) @ 0x15c7cb58 in /usr/bin/clickhouse 16. __tsan_thread_start_func @ 0x8a04ced in /usr/bin/clickhouse 17. start_thread @ 0x9609 in /usr/lib/x86_64-linux-gnu/libpthread-2.31.so 18. __clone @ 0x122293 in /usr/lib/x86_64-linux-gnu/libc-2.31.so (version 21.3.1.6130) [1]: https://clickhouse-test-reports.s3.yandex.net/21279/4f61ef3099f42f17b496a0b0424773978d9a32dc/stress_test_(thread).html#fail1 [2]: https://clickhouse-test-reports.s3.yandex.net/21292/ae9fea1d0af118a8f87b224d194d61da1567188b/stress_test_(thread).html#fail1 v2: https://clickhouse-test-reports.s3.yandex.net/21305/e969daa6e86c5e09cfef08cfde19712982b64e59/stress_test_(thread).html#fail1
2021-02-28 07:18:49 +00:00
do
2019-03-19 20:18:41 +00:00
{
Fix abnormal server termination for nested writers Writers with nested writer can call next() from the dtor for nested writer and this will cause exception again, so the buffer position should be updated on exceptions. Found by stress test (thread) here [1] and here [2]: 2021.02.27 19:27:53.498977 [ 302 ] {} <Fatal> BaseDaemon: (version 21.3.1.6130, build id: 2DAEC5DEBF03C5A1C3BF66B7779C886F16239345) (from thread 1614) Terminate called for uncaught exception: Code: 24, e.displayText() = DB::Exception: Cannot write to ostream at offset 2097498, Stack trace (when copying this message, always include the lines below): 0. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/exception:0: Poco::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) @ 0x15bef2ab in /usr/bin/clickhouse 1. ./obj-x86_64-linux-gnu/../src/Common/Exception.cpp:56: DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0x8aea92e in /usr/bin/clickhouse 2. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:0: DB::WriteBufferFromOStream::nextImpl() @ 0x8bbbc45 in /usr/bin/clickhouse 3. ./obj-x86_64-linux-gnu/../src/IO/BufferBase.h:39: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8bbc077 in /usr/bin/clickhouse 4. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:44: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8bbc18a in /usr/bin/clickhouse 5. ./obj-x86_64-linux-gnu/../src/IO/BufferWithOwnMemory.h:137: DB::ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer() @ 0x118bdc29 in /usr/bin/clickhouse 6. ./obj-x86_64-linux-gnu/../src/IO/ZstdDeflatingWriteBuffer.cpp:32: DB::ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer() @ 0x118be3ea in /usr/bin/clickhouse 7. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:0: DB::WriteBufferFromHTTPServerResponse::finalize() @ 0x12f1dceb in /usr/bin/clickhouse 8. ./obj-x86_64-linux-gnu/../src/Server/HTTPHandler.cpp:703: DB::HTTPHandler::trySendExceptionToClient(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, DB::HTTPServerRequest&, DB::HTTPServerResponse&, DB::HTTPHandler::Output&) @ 0x12e9fecc in /usr/bin/clickhouse 9. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/string:1444: DB::HTTPHandler::handleRequest(DB::HTTPServerRequest&, DB::HTTPServerResponse&) @ 0x12ea0d60 in /usr/bin/clickhouse 10. ./obj-x86_64-linux-gnu/../src/Server/HTTP/HTTPServerConnection.cpp:0: DB::HTTPServerConnection::run() @ 0x12f16db1 in /usr/bin/clickhouse 11. ./obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerConnection.cpp:57: Poco::Net::TCPServerConnection::start() @ 0x15b184f3 in /usr/bin/clickhouse 12. ./obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerDispatcher.cpp:0: Poco::Net::TCPServerDispatcher::run() @ 0x15b18c1f in /usr/bin/clickhouse 13. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/ScopedLock.h:36: Poco::PooledThread::run() @ 0x15c7fdb2 in /usr/bin/clickhouse 14. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread.cpp:56: Poco::(anonymous namespace)::RunnableHolder::run() @ 0x15c7e350 in /usr/bin/clickhouse 15. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/SharedPtr.h:277: Poco::ThreadImpl::runnableEntry(void*) @ 0x15c7cb58 in /usr/bin/clickhouse 16. __tsan_thread_start_func @ 0x8a04ced in /usr/bin/clickhouse 17. start_thread @ 0x9609 in /usr/lib/x86_64-linux-gnu/libpthread-2.31.so 18. __clone @ 0x122293 in /usr/lib/x86_64-linux-gnu/libc-2.31.so (version 21.3.1.6130) [1]: https://clickhouse-test-reports.s3.yandex.net/21279/4f61ef3099f42f17b496a0b0424773978d9a32dc/stress_test_(thread).html#fail1 [2]: https://clickhouse-test-reports.s3.yandex.net/21292/ae9fea1d0af118a8f87b224d194d61da1567188b/stress_test_(thread).html#fail1 v2: https://clickhouse-test-reports.s3.yandex.net/21305/e969daa6e86c5e09cfef08cfde19712982b64e59/stress_test_(thread).html#fail1
2021-02-28 07:18:49 +00:00
out->nextIfAtEnd();
out_data = reinterpret_cast<unsigned char *>(out->position());
out_capacity = out->buffer().end() - out->position();
int result = BrotliEncoderCompressStream(
brotli->state,
in_available ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
&in_available,
&in_data,
&out_capacity,
&out_data,
nullptr);
out->position() = out->buffer().end() - out_capacity;
if (result == 0)
{
throw Exception("brotli compress failed", ErrorCodes::BROTLI_WRITE_FAILED);
}
2019-03-19 20:18:41 +00:00
}
Fix abnormal server termination for nested writers Writers with nested writer can call next() from the dtor for nested writer and this will cause exception again, so the buffer position should be updated on exceptions. Found by stress test (thread) here [1] and here [2]: 2021.02.27 19:27:53.498977 [ 302 ] {} <Fatal> BaseDaemon: (version 21.3.1.6130, build id: 2DAEC5DEBF03C5A1C3BF66B7779C886F16239345) (from thread 1614) Terminate called for uncaught exception: Code: 24, e.displayText() = DB::Exception: Cannot write to ostream at offset 2097498, Stack trace (when copying this message, always include the lines below): 0. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/exception:0: Poco::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) @ 0x15bef2ab in /usr/bin/clickhouse 1. ./obj-x86_64-linux-gnu/../src/Common/Exception.cpp:56: DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0x8aea92e in /usr/bin/clickhouse 2. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:0: DB::WriteBufferFromOStream::nextImpl() @ 0x8bbbc45 in /usr/bin/clickhouse 3. ./obj-x86_64-linux-gnu/../src/IO/BufferBase.h:39: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8bbc077 in /usr/bin/clickhouse 4. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:44: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8bbc18a in /usr/bin/clickhouse 5. ./obj-x86_64-linux-gnu/../src/IO/BufferWithOwnMemory.h:137: DB::ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer() @ 0x118bdc29 in /usr/bin/clickhouse 6. ./obj-x86_64-linux-gnu/../src/IO/ZstdDeflatingWriteBuffer.cpp:32: DB::ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer() @ 0x118be3ea in /usr/bin/clickhouse 7. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:0: DB::WriteBufferFromHTTPServerResponse::finalize() @ 0x12f1dceb in /usr/bin/clickhouse 8. ./obj-x86_64-linux-gnu/../src/Server/HTTPHandler.cpp:703: DB::HTTPHandler::trySendExceptionToClient(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, DB::HTTPServerRequest&, DB::HTTPServerResponse&, DB::HTTPHandler::Output&) @ 0x12e9fecc in /usr/bin/clickhouse 9. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/string:1444: DB::HTTPHandler::handleRequest(DB::HTTPServerRequest&, DB::HTTPServerResponse&) @ 0x12ea0d60 in /usr/bin/clickhouse 10. ./obj-x86_64-linux-gnu/../src/Server/HTTP/HTTPServerConnection.cpp:0: DB::HTTPServerConnection::run() @ 0x12f16db1 in /usr/bin/clickhouse 11. ./obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerConnection.cpp:57: Poco::Net::TCPServerConnection::start() @ 0x15b184f3 in /usr/bin/clickhouse 12. ./obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerDispatcher.cpp:0: Poco::Net::TCPServerDispatcher::run() @ 0x15b18c1f in /usr/bin/clickhouse 13. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/ScopedLock.h:36: Poco::PooledThread::run() @ 0x15c7fdb2 in /usr/bin/clickhouse 14. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread.cpp:56: Poco::(anonymous namespace)::RunnableHolder::run() @ 0x15c7e350 in /usr/bin/clickhouse 15. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/SharedPtr.h:277: Poco::ThreadImpl::runnableEntry(void*) @ 0x15c7cb58 in /usr/bin/clickhouse 16. __tsan_thread_start_func @ 0x8a04ced in /usr/bin/clickhouse 17. start_thread @ 0x9609 in /usr/lib/x86_64-linux-gnu/libpthread-2.31.so 18. __clone @ 0x122293 in /usr/lib/x86_64-linux-gnu/libc-2.31.so (version 21.3.1.6130) [1]: https://clickhouse-test-reports.s3.yandex.net/21279/4f61ef3099f42f17b496a0b0424773978d9a32dc/stress_test_(thread).html#fail1 [2]: https://clickhouse-test-reports.s3.yandex.net/21292/ae9fea1d0af118a8f87b224d194d61da1567188b/stress_test_(thread).html#fail1 v2: https://clickhouse-test-reports.s3.yandex.net/21305/e969daa6e86c5e09cfef08cfde19712982b64e59/stress_test_(thread).html#fail1
2021-02-28 07:18:49 +00:00
while (in_available > 0);
}
catch (...)
{
/// Do not try to write next time after exception.
out->position() = out->buffer().begin();
throw;
}
}
2021-11-22 11:19:26 +00:00
void BrotliWriteBuffer::finalizeBefore()
Fix abnormal server termination for nested writers Writers with nested writer can call next() from the dtor for nested writer and this will cause exception again, so the buffer position should be updated on exceptions. Found by stress test (thread) here [1] and here [2]: 2021.02.27 19:27:53.498977 [ 302 ] {} <Fatal> BaseDaemon: (version 21.3.1.6130, build id: 2DAEC5DEBF03C5A1C3BF66B7779C886F16239345) (from thread 1614) Terminate called for uncaught exception: Code: 24, e.displayText() = DB::Exception: Cannot write to ostream at offset 2097498, Stack trace (when copying this message, always include the lines below): 0. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/exception:0: Poco::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) @ 0x15bef2ab in /usr/bin/clickhouse 1. ./obj-x86_64-linux-gnu/../src/Common/Exception.cpp:56: DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0x8aea92e in /usr/bin/clickhouse 2. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:0: DB::WriteBufferFromOStream::nextImpl() @ 0x8bbbc45 in /usr/bin/clickhouse 3. ./obj-x86_64-linux-gnu/../src/IO/BufferBase.h:39: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8bbc077 in /usr/bin/clickhouse 4. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:44: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8bbc18a in /usr/bin/clickhouse 5. ./obj-x86_64-linux-gnu/../src/IO/BufferWithOwnMemory.h:137: DB::ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer() @ 0x118bdc29 in /usr/bin/clickhouse 6. ./obj-x86_64-linux-gnu/../src/IO/ZstdDeflatingWriteBuffer.cpp:32: DB::ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer() @ 0x118be3ea in /usr/bin/clickhouse 7. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:0: DB::WriteBufferFromHTTPServerResponse::finalize() @ 0x12f1dceb in /usr/bin/clickhouse 8. ./obj-x86_64-linux-gnu/../src/Server/HTTPHandler.cpp:703: DB::HTTPHandler::trySendExceptionToClient(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, DB::HTTPServerRequest&, DB::HTTPServerResponse&, DB::HTTPHandler::Output&) @ 0x12e9fecc in /usr/bin/clickhouse 9. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/string:1444: DB::HTTPHandler::handleRequest(DB::HTTPServerRequest&, DB::HTTPServerResponse&) @ 0x12ea0d60 in /usr/bin/clickhouse 10. ./obj-x86_64-linux-gnu/../src/Server/HTTP/HTTPServerConnection.cpp:0: DB::HTTPServerConnection::run() @ 0x12f16db1 in /usr/bin/clickhouse 11. ./obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerConnection.cpp:57: Poco::Net::TCPServerConnection::start() @ 0x15b184f3 in /usr/bin/clickhouse 12. ./obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerDispatcher.cpp:0: Poco::Net::TCPServerDispatcher::run() @ 0x15b18c1f in /usr/bin/clickhouse 13. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/ScopedLock.h:36: Poco::PooledThread::run() @ 0x15c7fdb2 in /usr/bin/clickhouse 14. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread.cpp:56: Poco::(anonymous namespace)::RunnableHolder::run() @ 0x15c7e350 in /usr/bin/clickhouse 15. ./obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/SharedPtr.h:277: Poco::ThreadImpl::runnableEntry(void*) @ 0x15c7cb58 in /usr/bin/clickhouse 16. __tsan_thread_start_func @ 0x8a04ced in /usr/bin/clickhouse 17. start_thread @ 0x9609 in /usr/lib/x86_64-linux-gnu/libpthread-2.31.so 18. __clone @ 0x122293 in /usr/lib/x86_64-linux-gnu/libc-2.31.so (version 21.3.1.6130) [1]: https://clickhouse-test-reports.s3.yandex.net/21279/4f61ef3099f42f17b496a0b0424773978d9a32dc/stress_test_(thread).html#fail1 [2]: https://clickhouse-test-reports.s3.yandex.net/21292/ae9fea1d0af118a8f87b224d194d61da1567188b/stress_test_(thread).html#fail1 v2: https://clickhouse-test-reports.s3.yandex.net/21305/e969daa6e86c5e09cfef08cfde19712982b64e59/stress_test_(thread).html#fail1
2021-02-28 07:18:49 +00:00
{
2019-03-19 20:18:41 +00:00
next();
2019-03-19 20:18:41 +00:00
while (true)
{
out->nextIfAtEnd();
out_data = reinterpret_cast<unsigned char *>(out->position());
out_capacity = out->buffer().end() - out->position();
int result = BrotliEncoderCompressStream(
brotli->state,
2019-03-19 20:18:41 +00:00
BROTLI_OPERATION_FINISH,
&in_available,
&in_data,
&out_capacity,
&out_data,
nullptr);
out->position() = out->buffer().end() - out_capacity;
2019-03-19 20:18:41 +00:00
if (BrotliEncoderIsFinished(brotli->state))
{
return;
}
if (result == 0)
{
2019-04-10 16:12:31 +00:00
throw Exception("brotli compress failed", ErrorCodes::BROTLI_WRITE_FAILED);
}
}
}
}
#endif