Sending progress in HTTP headers (continued) [#CLICKHOUSE-32].

contrib/libpoco/Net/include/Poco/Net/HTTPServerResponse.h

@@ -67,12 +67,12 @@ public:
 		/// Must not be called after beginSend(), sendFile(), sendBuffer()
 		/// or redirect() has been called.
 
-	virtual std::ostream& beginSend() = 0;
+	virtual void beginSend(std::ostream * out_header_stream, std::ostream * out_body_stream) = 0;
 		/// Sends the response headers to the client
 		/// but do not finish headers with \r\n,
 		/// allowing to continue sending additional header fields.
 		///
-		/// Returns an output stream for sending the remaining headers
+		/// Returns an output streams for sending the remaining headers
 		/// and response body.
 		///
 		/// Must not be called after send(), sendFile(), sendBuffer()

contrib/libpoco/Net/include/Poco/Net/HTTPServerResponseImpl.h

@@ -65,12 +65,12 @@ public:
 		/// Must not be called after beginSend(), sendFile(), sendBuffer()
 		/// or redirect() has been called.
 
-	std::ostream& beginSend();
+	void beginSend(std::ostream * out_header_stream, std::ostream * out_body_stream);
 		/// Sends the response headers to the client
 		/// but do not finish headers with \r\n,
 		/// allowing to continue sending additional header fields.
 		///
-		/// Returns an output stream for sending the remaining headers
+		/// Returns an output streams for sending the remaining headers
 		/// and response body.
 		///
 		/// Must not be called after send(), sendFile(), sendBuffer()
@@ -126,6 +126,7 @@ private:
 	HTTPServerSession& _session;
 	HTTPServerRequestImpl* _pRequest;
 	std::ostream*      _pStream;
+	std::ostream*      _pHeaderStream;
 
 	friend class HTTPServerRequestImpl;
 };

contrib/libpoco/Net/src/HTTPServerResponseImpl.cpp

@@ -48,14 +48,18 @@ namespace Net {
 HTTPServerResponseImpl::HTTPServerResponseImpl(HTTPServerSession& session):
 	_session(session),
 	_pRequest(0),
-	_pStream(0)
+	_pStream(0),
+	_pHeaderStream(0)
 {
 }
 
 
 HTTPServerResponseImpl::~HTTPServerResponseImpl()
 {
-	delete _pStream;
+	if (_pHeaderStream && _pHeaderStream != _pStream)
+		delete _pHeaderStream;
+	if (_pStream)
+		delete _pStream;
 }
 
 
@@ -107,44 +111,38 @@ std::ostream& HTTPServerResponseImpl::send()
 }
 
 
-std::ostream& HTTPServerResponseImpl::beginSend()
+void HTTPServerResponseImpl::beginSend(std::ostream * out_header_stream, std::ostream * out_body_stream)
 {
 	poco_assert (!_pStream);
+	poco_assert (!_pHeaderStream);
 
 	if ((_pRequest && _pRequest->getMethod() == HTTPRequest::HTTP_HEAD) ||
 		getStatus() < 200 ||
 		getStatus() == HTTPResponse::HTTP_NO_CONTENT ||
 		getStatus() == HTTPResponse::HTTP_NOT_MODIFIED)
 	{
-		Poco::CountingOutputStream cs;
-		beginWrite(cs);
-		_pStream = new HTTPFixedLengthOutputStream(_session, cs.chars());
-		beginWrite(*_pStream);
+		throw Exception("HTTPServerResponse::beginSend is invalid for HEAD request");
 	}
 	else if (getChunkedTransferEncoding())
 	{
-		HTTPHeaderOutputStream hs(_session);
-		beginWrite(hs);
+		_pHeaderStream = new HTTPHeaderOutputStream(_session);
+		beginWrite(*_pHeaderStream);
 		_pStream = new HTTPChunkedOutputStream(_session);
 	}
 	else if (hasContentLength())
 	{
-		Poco::CountingOutputStream cs;
-		beginWrite(cs);
-#if defined(POCO_HAVE_INT64)
-		_pStream = new HTTPFixedLengthOutputStream(_session, getContentLength64() + cs.chars());
-#else
-		_pStream = new HTTPFixedLengthOutputStream(_session, getContentLength() + cs.chars());
-#endif
-		beginWrite(*_pStream);
+		throw Exception("HTTPServerResponse::beginSend is invalid for response with Content-Length header");
 	}
 	else
 	{
 		_pStream = new HTTPOutputStream(_session);
+		_pHeaderStream = _pStream;
 		setKeepAlive(false);
 		beginWrite(*_pStream);
 	}
-	return *_pStream;
+
+	out_header_stream = _pHeaderStream;
+	out_body_stream = _pStream;
 }
 
 

dbms/include/DB/IO/WriteBufferFromHTTPServerResponse.h

@@ -45,7 +45,9 @@ private:
 	ZlibCompressionMethod compression_method;
 	int compression_level = Z_DEFAULT_COMPRESSION;
 
-	std::ostream * response_ostr = nullptr;
+	std::ostream * response_body_ostr = nullptr;
+	std::ostream * response_header_ostr = nullptr;
+
 	std::experimental::optional<WriteBufferFromOStream> out_raw;
 	std::experimental::optional<ZlibDeflatingWriteBuffer> deflating_buf;
 
@@ -57,6 +59,8 @@ private:
 
 
 	/// Must be called under locked mutex.
+	/// This method send headers, if this was not done already,
+	///  but not finish them with \r\n, allowing to send more headers subsequently.
 	void startSendHeaders()
 	{
 		if (!out)
@@ -78,21 +82,22 @@ private:
 					throw Exception("Logical error: unknown compression method passed to WriteBufferFromHTTPServerResponse",
 						ErrorCodes::LOGICAL_ERROR);
 
-				response_ostr = &response.beginSend();
-				out_raw.emplace(*response_ostr);
+				response.beginSend(response_header_ostr, response_body_ostr);
+				out_raw.emplace(*response_body_ostr);
 				/// Use memory allocated for the outer buffer in the buffer pointed to by out. This avoids extra allocation and copy.
 				deflating_buf.emplace(out_raw.value(), compression_method, compression_level, working_buffer.size(), working_buffer.begin());
 				out = &deflating_buf.value();
 			}
 			else
 			{
-				response_ostr = &response.beginSend();
-				out_raw.emplace(*response_ostr, working_buffer.size(), working_buffer.begin());
+				response.beginSend(response_header_ostr, response_body_ostr);
+				out_raw.emplace(*response_body_ostr, working_buffer.size(), working_buffer.begin());
 				out = &out_raw.value();
 			}
 		}
 	}
 
+	/// This method send headers, if this was not done already, and finish them with \r\n, allowing to start to send body.
 	void sendAllHeaders()
 	{
 		startSendHeaders();
@@ -101,7 +106,7 @@ private:
 		{
 			body_started_sending = true;
 			/// Send end of headers delimiter.
-			*response_ostr << "\r\n" << std::flush;
+			*response_header_ostr << "\r\n" << std::flush;
 		}
 	}
 
@@ -136,6 +141,7 @@ public:
 		if (body_started_sending)
 			return;
 
+		/// Send all common headers before our special progress headers.
 		startSendHeaders();
 
 		std::string progress_string;
@@ -144,7 +150,7 @@ public:
 			progress.writeJSON(progress_string_writer);
 		}
 
-		*response_ostr << "X-ClickHouse-Progress: " << progress_string << "\r\n" << std::flush;
+		*response_header_ostr << "X-ClickHouse-Progress: " << progress_string << "\r\n" << std::flush;
 	}
 
 	/// Send at least HTTP headers if no data has been sent yet.