From 3b0900e6cf6f4179d8ba7b2738ef201cab53b614 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 30 Jan 2017 08:13:58 +0300 Subject: [PATCH 1/4] Moved code to .cpp [#CLICKHOUSE-2027]. --- dbms/include/DB/IO/HTTPCommon.h | 10 +- .../DB/IO/WriteBufferFromHTTPServerResponse.h | 140 ++-------------- dbms/src/IO/HTTPCommon.cpp | 2 + .../IO/WriteBufferFromHTTPServerResponse.cpp | 156 ++++++++++++++++++ 4 files changed, 182 insertions(+), 126 deletions(-) create mode 100644 dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp diff --git a/dbms/include/DB/IO/HTTPCommon.h b/dbms/include/DB/IO/HTTPCommon.h index 76683e543c8..84de0761d09 100644 --- a/dbms/include/DB/IO/HTTPCommon.h +++ b/dbms/include/DB/IO/HTTPCommon.h @@ -1,6 +1,14 @@ #pragma once -#include + +namespace Poco +{ + namespace Net + { + class HTTPServerResponse; + } +} + namespace DB { diff --git a/dbms/include/DB/IO/WriteBufferFromHTTPServerResponse.h b/dbms/include/DB/IO/WriteBufferFromHTTPServerResponse.h index a3cbdfeb9a9..ba47fda82c2 100644 --- a/dbms/include/DB/IO/WriteBufferFromHTTPServerResponse.h +++ b/dbms/include/DB/IO/WriteBufferFromHTTPServerResponse.h @@ -3,14 +3,9 @@ #include #include -#include - -#include - #include #include #include -#include #include #include #include @@ -18,15 +13,18 @@ #include -namespace DB +namespace Poco { - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; + namespace Net + { + class HTTPServerResponse; + } } +namespace DB +{ + /// The difference from WriteBufferFromOStream is that this buffer gets the underlying std::ostream /// (using response.send()) only after data is flushed for the first time. This is needed in HTTP /// servers to change some HTTP headers (e.g. response code) before any data is sent to the client @@ -71,126 +69,28 @@ private: /// Must be called under locked mutex. /// This method send headers, if this was not done already, /// but not finish them with \r\n, allowing to send more headers subsequently. - void startSendHeaders() - { - if (!headers_started_sending) - { - headers_started_sending = true; - - if (add_cors_header) - response.set("Access-Control-Allow-Origin", "*"); - - setResponseDefaultHeaders(response); - - std::tie(response_header_ostr, response_body_ostr) = response.beginSend(); - } - } + void startSendHeaders(); /// This method finish headers with \r\n, allowing to start to send body. - void finishSendHeaders() - { - if (!headers_finished_sending) - { - headers_finished_sending = true; + void finishSendHeaders(); - /// Send end of headers delimiter. - *response_header_ostr << "\r\n" << std::flush; - } - } - - void nextImpl() override - { - { - std::lock_guard lock(mutex); - - startSendHeaders(); - - if (!out) - { - if (compress) - { - if (compression_method == ZlibCompressionMethod::Gzip) - *response_header_ostr << "Content-Encoding: gzip\r\n"; - else if (compression_method == ZlibCompressionMethod::Zlib) - *response_header_ostr << "Content-Encoding: deflate\r\n"; - else - throw Exception("Logical error: unknown compression method passed to WriteBufferFromHTTPServerResponse", - ErrorCodes::LOGICAL_ERROR); - - /// Use memory allocated for the outer buffer in the buffer pointed to by out. This avoids extra allocation and copy. - out_raw.emplace(*response_body_ostr); - deflating_buf.emplace(out_raw.value(), compression_method, compression_level, working_buffer.size(), working_buffer.begin()); - out = &deflating_buf.value(); - } - else - { - out_raw.emplace(*response_body_ostr, working_buffer.size(), working_buffer.begin()); - out = &out_raw.value(); - } - } - - finishSendHeaders(); - } - - out->position() = position(); - out->next(); - } + void nextImpl() override; public: WriteBufferFromHTTPServerResponse( Poco::Net::HTTPServerResponse & response_, bool compress_ = false, /// If true - set Content-Encoding header and compress the result. ZlibCompressionMethod compression_method_ = ZlibCompressionMethod::Gzip, - size_t size = DBMS_DEFAULT_BUFFER_SIZE) - : BufferWithOwnMemory(size), response(response_), - compress(compress_), compression_method(compression_method_) {} + size_t size = DBMS_DEFAULT_BUFFER_SIZE); /// Writes progess in repeating HTTP headers. - void onProgress(const Progress & progress) - { - std::lock_guard lock(mutex); - - /// Cannot add new headers if body was started to send. - if (headers_finished_sending) - return; - - accumulated_progress.incrementPiecewiseAtomically(progress); - - if (progress_watch.elapsed() >= send_progress_interval_ms * 1000000) - { - progress_watch.restart(); - - /// Send all common headers before our special progress headers. - startSendHeaders(); - - std::string progress_string; - { - WriteBufferFromString progress_string_writer(progress_string); - accumulated_progress.writeJSON(progress_string_writer); - } - - *response_header_ostr << "X-ClickHouse-Progress: " << progress_string << "\r\n" << std::flush; - } - } + void onProgress(const Progress & progress); /// Send at least HTTP headers if no data has been sent yet. /// Use after the data has possibly been sent and no error happened (and thus you do not plan /// to change response HTTP code. /// This method is idempotent. - void finalize() - { - if (offset()) - { - next(); - } - else - { - /// If no remaining data, just send headers. - std::lock_guard lock(mutex); - startSendHeaders(); - finishSendHeaders(); - } - } + void finalize(); /// Turn compression on or off. /// The setting has any effect only if HTTP headers haven't been sent yet. @@ -219,17 +119,7 @@ public: send_progress_interval_ms = send_progress_interval_ms_; } - ~WriteBufferFromHTTPServerResponse() - { - try - { - finalize(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } + ~WriteBufferFromHTTPServerResponse(); }; } diff --git a/dbms/src/IO/HTTPCommon.cpp b/dbms/src/IO/HTTPCommon.cpp index c13ea04968f..9bc2a931476 100644 --- a/dbms/src/IO/HTTPCommon.cpp +++ b/dbms/src/IO/HTTPCommon.cpp @@ -1,6 +1,8 @@ #include #include +#include + namespace DB { diff --git a/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp b/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp new file mode 100644 index 00000000000..2ddcb275f74 --- /dev/null +++ b/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp @@ -0,0 +1,156 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + +void WriteBufferFromHTTPServerResponse::startSendHeaders() +{ + if (!headers_started_sending) + { + headers_started_sending = true; + + if (add_cors_header) + response.set("Access-Control-Allow-Origin", "*"); + + setResponseDefaultHeaders(response); + + std::tie(response_header_ostr, response_body_ostr) = response.beginSend(); + } +} + + +void WriteBufferFromHTTPServerResponse::finishSendHeaders() +{ + if (!headers_finished_sending) + { + headers_finished_sending = true; + + /// Send end of headers delimiter. + *response_header_ostr << "\r\n" << std::flush; + } +} + + +void WriteBufferFromHTTPServerResponse::nextImpl() +{ + { + std::lock_guard lock(mutex); + + startSendHeaders(); + + if (!out) + { + if (compress) + { + if (compression_method == ZlibCompressionMethod::Gzip) + *response_header_ostr << "Content-Encoding: gzip\r\n"; + else if (compression_method == ZlibCompressionMethod::Zlib) + *response_header_ostr << "Content-Encoding: deflate\r\n"; + else + throw Exception("Logical error: unknown compression method passed to WriteBufferFromHTTPServerResponse", + ErrorCodes::LOGICAL_ERROR); + + /// Use memory allocated for the outer buffer in the buffer pointed to by out. This avoids extra allocation and copy. + out_raw.emplace(*response_body_ostr); + deflating_buf.emplace(out_raw.value(), compression_method, compression_level, working_buffer.size(), working_buffer.begin()); + out = &deflating_buf.value(); + } + else + { + out_raw.emplace(*response_body_ostr, working_buffer.size(), working_buffer.begin()); + out = &out_raw.value(); + } + } + + finishSendHeaders(); + } + + out->position() = position(); + out->next(); +} + + +WriteBufferFromHTTPServerResponse::WriteBufferFromHTTPServerResponse( + Poco::Net::HTTPServerResponse & response_, + bool compress_, + ZlibCompressionMethod compression_method_, + size_t size) + : BufferWithOwnMemory(size), response(response_), + compress(compress_), compression_method(compression_method_) +{ +} + + +void WriteBufferFromHTTPServerResponse::onProgress(const Progress & progress) +{ + std::lock_guard lock(mutex); + + /// Cannot add new headers if body was started to send. + if (headers_finished_sending) + return; + + accumulated_progress.incrementPiecewiseAtomically(progress); + + if (progress_watch.elapsed() >= send_progress_interval_ms * 1000000) + { + progress_watch.restart(); + + /// Send all common headers before our special progress headers. + startSendHeaders(); + + std::string progress_string; + { + WriteBufferFromString progress_string_writer(progress_string); + accumulated_progress.writeJSON(progress_string_writer); + } + + *response_header_ostr << "X-ClickHouse-Progress: " << progress_string << "\r\n" << std::flush; + } +} + + +void WriteBufferFromHTTPServerResponse::finalize() +{ + if (offset()) + { + next(); + } + else + { + /// If no remaining data, just send headers. + std::lock_guard lock(mutex); + startSendHeaders(); + finishSendHeaders(); + } +} + + +WriteBufferFromHTTPServerResponse::~WriteBufferFromHTTPServerResponse() +{ + try + { + finalize(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } +} + +} From 2c488b7cbc9b138c1bc8329a63a092c113e3deff Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Mon, 30 Jan 2017 07:32:19 +0300 Subject: [PATCH 2/4] CMakeLists.txt: "-no-pie" should be "---no-pie" --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d1cf8c35484..34eec132a8e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,8 +126,8 @@ if (NOT APPLE) endif () if (USE_STATIC_LIBRARIES AND NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -no-pie") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -no-pie") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --no-pie") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --no-pie") endif () From ecc6d2a7b93910a564dfcb366b116c3144364bb0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 30 Jan 2017 08:22:49 +0300 Subject: [PATCH 3/4] Moved a function to header for inlining [#CLICKHOUSE-2]. --- dbms/include/DB/Columns/ColumnNullable.h | 8 +++++++- dbms/src/Columns/ColumnNullable.cpp | 6 ------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dbms/include/DB/Columns/ColumnNullable.h b/dbms/include/DB/Columns/ColumnNullable.h index 954dce20632..019d4436f8d 100644 --- a/dbms/include/DB/Columns/ColumnNullable.h +++ b/dbms/include/DB/Columns/ColumnNullable.h @@ -40,7 +40,13 @@ public: void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;; void insert(const Field & x) override; void insertFrom(const IColumn & src, size_t n) override; - void insertDefault() override; + + void insertDefault() override + { + nested_column->insertDefault(); + getNullMap().push_back(1); + } + void popBack(size_t n) override; ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; ColumnPtr permute(const Permutation & perm, size_t limit) const override; diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index 20d519308fe..6e320b1b45f 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -148,12 +148,6 @@ void ColumnNullable::insertFrom(const IColumn & src, size_t n) getNullMap().push_back(src_concrete.getNullMap()[n]); } -void ColumnNullable::insertDefault() -{ - nested_column->insertDefault(); - getNullMap().push_back(1); -} - void ColumnNullable::popBack(size_t n) { nested_column->popBack(n); From fbe888c224fbea9b03ecfc58ed03dbbe7c6fac8d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 30 Jan 2017 08:27:26 +0300 Subject: [PATCH 4/4] Manual version update [#CLICKHOUSE-2]. --- libs/libcommon/cmake/version.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/libcommon/cmake/version.cmake b/libs/libcommon/cmake/version.cmake index 20bff3f2e3d..ba09d8487f6 100644 --- a/libs/libcommon/cmake/version.cmake +++ b/libs/libcommon/cmake/version.cmake @@ -1,6 +1,6 @@ #This strings autochanged from release_lib.sh : -set(VERSION_DESCRIBE v1.1.54134-testing) -set(VERSION_REVISION 54139) +set(VERSION_DESCRIBE v1.1.54140-testing) +set(VERSION_REVISION 54140) #===end of autochange set(VERSION_MAJOR 1)