thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 09:02:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

handle brotli errors; working wersion of BrotliReadBuffer

Browse Source
This commit is contained in:
Mikhail Fandyushin 2019-02-06 02:12:31 +03:00
parent 7e9e34bebf
commit 1ee18203ae
3 changed files with 69 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
dbms/programs/server/HTTPHandler.cpp
View File

@ -397,25 +397,23 @@ void HTTPHandler::processQuery(
String http_request_compression_method_str = request.get("Content-Encoding", "");
if (!http_request_compression_method_str.empty())
{
do {
if (http_request_compression_method_str == "gzip")
{
in_post = std::make_unique<ZlibInflatingReadBuffer>(*in_post_raw, ZlibCompressionMethod::Gzip);
break;
}
if (http_request_compression_method_str == "deflate")
{
in_post = std::make_unique<ZlibInflatingReadBuffer>(*in_post_raw, ZlibCompressionMethod::Zlib);
break;
}
if (http_request_compression_method_str == "br")
{
in_post = std::make_unique<BrotliReadBuffer>(*in_post_raw);
break;
}
if (http_request_compression_method_str == "gzip")
{
in_post = std::make_unique<ZlibInflatingReadBuffer>(*in_post_raw, ZlibCompressionMethod::Gzip);
}
else if (http_request_compression_method_str == "deflate")
{
in_post = std::make_unique<ZlibInflatingReadBuffer>(*in_post_raw, ZlibCompressionMethod::Zlib);
}
else if (http_request_compression_method_str == "br")
{
in_post = std::make_unique<BrotliReadBuffer>(*in_post_raw);
}
else
{
throw Exception("Unknown Content-Encoding of HTTP request: " + http_request_compression_method_str,
ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
} while(0);
ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
}
else
in_post = std::move(in_post_raw);

51
dbms/src/IO/BrotliReadBuffer.cpp
View File

@ -5,14 +5,19 @@ namespace DB
BrotliReadBuffer::BrotliReadBuffer(ReadBuffer &in_, size_t buf_size, char *existing_memory, size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment)
, in(in_)
, bstate(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr))
, bresult(BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT)
, in_available(0)
, in_data(nullptr)
, out_capacity(0)
, out_data(nullptr)
, eof(false)
{
bstate_ = BrotliDecoderCreateInstance(NULL,NULL,NULL);
}
BrotliReadBuffer::~BrotliReadBuffer()
{
BrotliDecoderDestroyInstance(bstate_);
BrotliDecoderDestroyInstance(bstate);
}
bool BrotliReadBuffer::nextImpl()
@ -20,20 +25,42 @@ bool BrotliReadBuffer::nextImpl()
if (eof)
return false;
auto ptr_in = reinterpret_cast<const uint8_t *>(in.position());
size_t size_in = in.buffer().end() - in.position();
if (!in_available)
{
in.nextIfAtEnd();
in_available = in.buffer().end() - in.position();
in_data = reinterpret_cast<uint8_t *>(in.position());
}
auto ptr_out = reinterpret_cast<uint8_t *>(internal_buffer.begin());
size_t size_out = internal_buffer.size();
if (bresult == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && (!in_available || in.eof()))
{
throw Exception(std::string("brotli decode error"), ErrorCodes::CANNOT_READ_ALL_DATA);
}
BrotliDecoderDecompressStream(bstate_, &size_in, &ptr_in, &size_out, &ptr_out, nullptr);
out_capacity = internal_buffer.size();
out_data = reinterpret_cast<uint8_t *>(internal_buffer.begin());
in.position() = in.buffer().end() - size_in;
working_buffer.resize(internal_buffer.size() - size_out);
bresult = BrotliDecoderDecompressStream(bstate, &in_available, &in_data, &out_capacity, &out_data, nullptr);
if (in.eof()) {
eof = true;
return working_buffer.size() != 0;
in.position() = in.buffer().end() - in_available;
working_buffer.resize(internal_buffer.size() - out_capacity);
if (bresult == BROTLI_DECODER_RESULT_SUCCESS)
{
if (in.eof())
{
eof = true;
return working_buffer.size() != 0;
}
else
{
throw Exception(std::string("brotli decode error"), ErrorCodes::CANNOT_READ_ALL_DATA);
}
}
if (bresult == BROTLI_DECODER_RESULT_ERROR)
{
throw Exception(std::string("brotli decode error"), ErrorCodes::CANNOT_READ_ALL_DATA);
}
return true;

17
dbms/src/IO/BrotliReadBuffer.h
View File

@ -5,9 +5,11 @@
#include <brotli/decode.h>
namespace DB {
namespace DB
{
class BrotliReadBuffer : public BufferWithOwnMemory<ReadBuffer> {
class BrotliReadBuffer : public BufferWithOwnMemory<ReadBuffer>
{
public:
BrotliReadBuffer(
ReadBuffer &in_,
@ -21,7 +23,16 @@ private:
bool nextImpl() override;
ReadBuffer &in;
BrotliDecoderState * bstate_;
BrotliDecoderState * bstate;
BrotliDecoderResult bresult;
size_t in_available;
const uint8_t * in_data;
size_t out_capacity;
uint8_t * out_data;
bool eof;
};
}