From e1d0994c0a652aae2c758e8577506b5fc08e33ef Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Mon, 30 Oct 2023 05:37:47 +0000 Subject: [PATCH] fix --- src/IO/BrotliWriteBuffer.cpp | 2 +- src/IO/BrotliWriteBuffer.h | 5 ++++- src/IO/Bzip2WriteBuffer.cpp | 2 +- src/IO/Bzip2WriteBuffer.h | 8 +++++--- src/IO/CompressionMethod.cpp | 24 +++++++++++++----------- src/IO/CompressionMethod.h | 6 ++++-- src/IO/LZMADeflatingWriteBuffer.cpp | 2 +- src/IO/LZMADeflatingWriteBuffer.h | 7 +++++-- src/IO/Lz4DeflatingWriteBuffer.cpp | 2 +- src/IO/Lz4DeflatingWriteBuffer.h | 5 ++++- src/IO/ZlibDeflatingWriteBuffer.cpp | 2 +- src/IO/ZlibDeflatingWriteBuffer.h | 6 ++++-- src/IO/ZstdDeflatingWriteBuffer.cpp | 2 +- src/IO/ZstdDeflatingWriteBuffer.h | 6 ++++-- src/Server/HTTPHandler.cpp | 7 ++++++- 15 files changed, 55 insertions(+), 31 deletions(-) diff --git a/src/IO/BrotliWriteBuffer.cpp b/src/IO/BrotliWriteBuffer.cpp index 21807ecd503..4c7484f02a4 100644 --- a/src/IO/BrotliWriteBuffer.cpp +++ b/src/IO/BrotliWriteBuffer.cpp @@ -74,7 +74,7 @@ void BrotliWriteBuffer::finalizeBefore() next(); /// Don't write out if no data was ever compressed - if (total_out == 0) + if (!compress_empty && total_out == 0) return; while (true) diff --git a/src/IO/BrotliWriteBuffer.h b/src/IO/BrotliWriteBuffer.h index 8b90fca960f..e8bcf24d646 100644 --- a/src/IO/BrotliWriteBuffer.h +++ b/src/IO/BrotliWriteBuffer.h @@ -22,13 +22,15 @@ public: int compression_level, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, - size_t alignment = 0) + size_t alignment = 0, + bool compress_empty_ = true) : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment) , brotli(std::make_unique()) , in_available(0) , in_data(nullptr) , out_capacity(0) , out_data(nullptr) + , compress_empty(compress_empty_) { BrotliEncoderSetParameter(brotli->state, BROTLI_PARAM_QUALITY, static_cast(compression_level)); // Set LZ77 window size. According to brotli sources default value is 24 (c/tools/brotli.c:81) @@ -62,6 +64,7 @@ private: protected: size_t total_out = 0; + bool compress_empty = true; }; } diff --git a/src/IO/Bzip2WriteBuffer.cpp b/src/IO/Bzip2WriteBuffer.cpp index 9463cbe6c6a..8c9a6d15921 100644 --- a/src/IO/Bzip2WriteBuffer.cpp +++ b/src/IO/Bzip2WriteBuffer.cpp @@ -79,7 +79,7 @@ void Bzip2WriteBuffer::finalizeBefore() next(); /// Don't write out if no data was ever compressed - if (bz->stream.total_out_hi32 == 0 && bz->stream.total_out_lo32 == 0) + if (!compress_empty && bz->stream.total_out_hi32 == 0 && bz->stream.total_out_lo32 == 0) return; out->nextIfAtEnd(); diff --git a/src/IO/Bzip2WriteBuffer.h b/src/IO/Bzip2WriteBuffer.h index eac5e9df428..b083f63ae8e 100644 --- a/src/IO/Bzip2WriteBuffer.h +++ b/src/IO/Bzip2WriteBuffer.h @@ -21,9 +21,10 @@ public: int compression_level, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, - size_t alignment = 0) - : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment) - , bz(std::make_unique(compression_level)) + size_t alignment = 0, + bool compress_empty_ = true) + : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment), bz(std::make_unique(compression_level)) + , compress_empty(compress_empty_) { } @@ -44,6 +45,7 @@ private: }; std::unique_ptr bz; + bool compress_empty = true; }; } diff --git a/src/IO/CompressionMethod.cpp b/src/IO/CompressionMethod.cpp index 1101aaccebe..90453e16961 100644 --- a/src/IO/CompressionMethod.cpp +++ b/src/IO/CompressionMethod.cpp @@ -172,27 +172,27 @@ std::unique_ptr wrapReadBufferWithCompressionMethod( template std::unique_ptr createWriteCompressedWrapper( - WriteBufferT && nested, CompressionMethod method, int level, size_t buf_size, char * existing_memory, size_t alignment) + WriteBufferT && nested, CompressionMethod method, int level, size_t buf_size, char * existing_memory, size_t alignment, bool compress_empty) { if (method == DB::CompressionMethod::Gzip || method == CompressionMethod::Zlib) - return std::make_unique(std::forward(nested), method, level, buf_size, existing_memory, alignment); + return std::make_unique(std::forward(nested), method, level, buf_size, existing_memory, alignment, compress_empty); #if USE_BROTLI if (method == DB::CompressionMethod::Brotli) - return std::make_unique(std::forward(nested), level, buf_size, existing_memory, alignment); + return std::make_unique(std::forward(nested), level, buf_size, existing_memory, alignment, compress_empty); #endif if (method == CompressionMethod::Xz) - return std::make_unique(std::forward(nested), level, buf_size, existing_memory, alignment); + return std::make_unique(std::forward(nested), level, buf_size, existing_memory, alignment, compress_empty); if (method == CompressionMethod::Zstd) - return std::make_unique(std::forward(nested), level, buf_size, existing_memory, alignment); + return std::make_unique(std::forward(nested), level, buf_size, existing_memory, alignment, compress_empty); if (method == CompressionMethod::Lz4) - return std::make_unique(std::forward(nested), level, buf_size, existing_memory, alignment); + return std::make_unique(std::forward(nested), level, buf_size, existing_memory, alignment, compress_empty); #if USE_BZIP2 if (method == CompressionMethod::Bzip2) - return std::make_unique(std::forward(nested), level, buf_size, existing_memory, alignment); + return std::make_unique(std::forward(nested), level, buf_size, existing_memory, alignment, compress_empty); #endif #if USE_SNAPPY if (method == CompressionMethod::Snappy) @@ -209,11 +209,12 @@ std::unique_ptr wrapWriteBufferWithCompressionMethod( int level, size_t buf_size, char * existing_memory, - size_t alignment) + size_t alignment, + bool compress_empty) { if (method == CompressionMethod::None) return nested; - return createWriteCompressedWrapper(nested, method, level, buf_size, existing_memory, alignment); + return createWriteCompressedWrapper(nested, method, level, buf_size, existing_memory, alignment, compress_empty); } @@ -223,10 +224,11 @@ std::unique_ptr wrapWriteBufferWithCompressionMethod( int level, size_t buf_size, char * existing_memory, - size_t alignment) + size_t alignment, + bool compress_empty) { assert(method != CompressionMethod::None); - return createWriteCompressedWrapper(nested, method, level, buf_size, existing_memory, alignment); + return createWriteCompressedWrapper(nested, method, level, buf_size, existing_memory, alignment, compress_empty); } } diff --git a/src/IO/CompressionMethod.h b/src/IO/CompressionMethod.h index 7aa68ecea7b..e9e334a6922 100644 --- a/src/IO/CompressionMethod.h +++ b/src/IO/CompressionMethod.h @@ -79,7 +79,8 @@ std::unique_ptr wrapWriteBufferWithCompressionMethod( int level, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, - size_t alignment = 0); + size_t alignment = 0, + bool compress_empty = true); std::unique_ptr wrapWriteBufferWithCompressionMethod( WriteBuffer * nested, @@ -87,6 +88,7 @@ std::unique_ptr wrapWriteBufferWithCompressionMethod( int level, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, - size_t alignment = 0); + size_t alignment = 0, + bool compress_empty = true); } diff --git a/src/IO/LZMADeflatingWriteBuffer.cpp b/src/IO/LZMADeflatingWriteBuffer.cpp index d471f69a8c0..db8f8c95fe6 100644 --- a/src/IO/LZMADeflatingWriteBuffer.cpp +++ b/src/IO/LZMADeflatingWriteBuffer.cpp @@ -93,7 +93,7 @@ void LZMADeflatingWriteBuffer::finalizeBefore() next(); /// Don't write out if no data was ever compressed - if (lstr.total_out) + if (!compress_empty && lstr.total_out == 0) return; do diff --git a/src/IO/LZMADeflatingWriteBuffer.h b/src/IO/LZMADeflatingWriteBuffer.h index 355872bf0f0..797b85cd400 100644 --- a/src/IO/LZMADeflatingWriteBuffer.h +++ b/src/IO/LZMADeflatingWriteBuffer.h @@ -20,8 +20,9 @@ public: int compression_level, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, - size_t alignment = 0) - : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment) + size_t alignment = 0, + bool compress_empty_ = true) + : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment), compress_empty(compress_empty_) { initialize(compression_level); } @@ -37,6 +38,8 @@ private: void finalizeAfter() override; lzma_stream lstr; + + bool compress_empty = true; }; } diff --git a/src/IO/Lz4DeflatingWriteBuffer.cpp b/src/IO/Lz4DeflatingWriteBuffer.cpp index d30137ac8fd..7b9ca2aeabf 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.cpp +++ b/src/IO/Lz4DeflatingWriteBuffer.cpp @@ -128,7 +128,7 @@ void Lz4DeflatingWriteBuffer::finalizeBefore() next(); /// Don't write out if no data was ever compressed - if (first_time) + if (!compress_empty && first_time) return; out_capacity = out->buffer().end() - out->position(); diff --git a/src/IO/Lz4DeflatingWriteBuffer.h b/src/IO/Lz4DeflatingWriteBuffer.h index 3ac0e4b671f..1d1bec92e23 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.h +++ b/src/IO/Lz4DeflatingWriteBuffer.h @@ -20,12 +20,14 @@ public: int compression_level, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, - size_t alignment = 0) + size_t alignment = 0, + bool compress_empty_ = true) : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment) , in_data(nullptr) , out_data(nullptr) , in_capacity(0) , out_capacity(0) + , compress_empty(compress_empty_) { initialize(compression_level); } @@ -50,5 +52,6 @@ private: size_t out_capacity; bool first_time = true; + bool compress_empty = true; }; } diff --git a/src/IO/ZlibDeflatingWriteBuffer.cpp b/src/IO/ZlibDeflatingWriteBuffer.cpp index 12e453121cb..ab6763fe6a6 100644 --- a/src/IO/ZlibDeflatingWriteBuffer.cpp +++ b/src/IO/ZlibDeflatingWriteBuffer.cpp @@ -53,7 +53,7 @@ void ZlibDeflatingWriteBuffer::finalizeBefore() next(); /// Don't write out if no data was ever compressed - if (zstr.total_out == 0) + if (!compress_empty && zstr.total_out == 0) return; /// https://github.com/zlib-ng/zlib-ng/issues/494 diff --git a/src/IO/ZlibDeflatingWriteBuffer.h b/src/IO/ZlibDeflatingWriteBuffer.h index 5e291e1f6e0..f01c41c7d13 100644 --- a/src/IO/ZlibDeflatingWriteBuffer.h +++ b/src/IO/ZlibDeflatingWriteBuffer.h @@ -28,8 +28,9 @@ public: int compression_level, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, - size_t alignment = 0) - : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment) + size_t alignment = 0, + bool compress_empty_ = true) + : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment), compress_empty(compress_empty_) { zstr.zalloc = nullptr; zstr.zfree = nullptr; @@ -63,6 +64,7 @@ private: virtual void finalizeAfter() override; z_stream zstr; + bool compress_empty = true; }; } diff --git a/src/IO/ZstdDeflatingWriteBuffer.cpp b/src/IO/ZstdDeflatingWriteBuffer.cpp index a0b051340f2..bad6e733cf1 100644 --- a/src/IO/ZstdDeflatingWriteBuffer.cpp +++ b/src/IO/ZstdDeflatingWriteBuffer.cpp @@ -86,7 +86,7 @@ void ZstdDeflatingWriteBuffer::nextImpl() void ZstdDeflatingWriteBuffer::finalizeBefore() { /// Don't write out if no data was ever compressed - if (total_out == 0) + if (!compress_empty && total_out == 0) return; flush(ZSTD_e_end); } diff --git a/src/IO/ZstdDeflatingWriteBuffer.h b/src/IO/ZstdDeflatingWriteBuffer.h index 0d04a43e9a8..d25db515d28 100644 --- a/src/IO/ZstdDeflatingWriteBuffer.h +++ b/src/IO/ZstdDeflatingWriteBuffer.h @@ -20,8 +20,9 @@ public: int compression_level, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, - size_t alignment = 0) - : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment) + size_t alignment = 0, + bool compress_empty_ = true) + : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment), compress_empty(compress_empty_) { initialize(compression_level); } @@ -51,6 +52,7 @@ private: ZSTD_outBuffer output; size_t total_out = 0; + bool compress_empty = true; }; } diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 29f18b8da76..23de513c86b 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -630,7 +630,12 @@ void HTTPHandler::processQuery( if (client_supports_http_compression && enable_http_compression) { used_output.out_holder->setCompressionMethodHeader(http_response_compression_method); - used_output.wrap_compressed_holder = wrapWriteBufferWithCompressionMethod(used_output.out_holder.get(), http_response_compression_method, static_cast(http_zlib_compression_level)); + used_output.wrap_compressed_holder = + wrapWriteBufferWithCompressionMethod( + used_output.out_holder.get(), + http_response_compression_method, + static_cast(http_zlib_compression_level), + DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0, false); used_output.out = used_output.wrap_compressed_holder; } else