thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

stash

Browse Source
This commit is contained in:
Nikita Taranov 2024-03-22 16:09:14 +00:00
parent 2de9697c8f
commit d7b34a80bb
22 changed files with 63 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
base/poco/Net/include/Poco/Net/HTTPServerSession.h
View File

@ -56,10 +56,15 @@ namespace Net
SocketAddress serverAddress();
/// Returns the server's address.
size_t getKeepAliveTimeout() const { return _params->getKeepAliveTimeout().totalSeconds(); }
size_t getMaxKeepAliveRequests() const { return _params->getMaxKeepAliveRequests(); }
private:
bool _firstRequest;
Poco::Timespan _keepAliveTimeout;
int _maxKeepAliveRequests;
HTTPServerParams::Ptr _params;
};

22
base/poco/Net/src/HTTPServerSession.cpp
View File

@ -19,11 +19,12 @@ namespace Poco {
namespace Net {
HTTPServerSession::HTTPServerSession(const StreamSocket& socket, HTTPServerParams::Ptr pParams):
HTTPSession(socket, pParams->getKeepAlive()),
_firstRequest(true),
_keepAliveTimeout(pParams->getKeepAliveTimeout()),
_maxKeepAliveRequests(pParams->getMaxKeepAliveRequests())
HTTPServerSession::HTTPServerSession(const StreamSocket & socket, HTTPServerParams::Ptr pParams)
: HTTPSession(socket, pParams->getKeepAlive())
, _firstRequest(true)
, _keepAliveTimeout(pParams->getKeepAliveTimeout())
, _maxKeepAliveRequests(pParams->getMaxKeepAliveRequests())
, _params(pParams)
{
setTimeout(pParams->getTimeout());
}
@ -46,11 +47,12 @@ bool HTTPServerSession::hasMoreRequests()
}
else if (_maxKeepAliveRequests != 0 && getKeepAlive())
{
if (_maxKeepAliveRequests > 0)
--_maxKeepAliveRequests;
return buffered() > 0 || socket().poll(_keepAliveTimeout, Socket::SELECT_READ);
}
else return false;
if (_maxKeepAliveRequests > 0)
--_maxKeepAliveRequests;
return buffered() > 0 || socket().poll(_keepAliveTimeout, Socket::SELECT_READ);
}
else
return false;
}

16
programs/library-bridge/LibraryBridgeHandlers.cpp
View File

@ -374,10 +374,8 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ
}
ExternalDictionaryLibraryBridgeExistsHandler::ExternalDictionaryLibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_)
: WithContext(context_)
, keep_alive_timeout(keep_alive_timeout_)
, log(getLogger("ExternalDictionaryLibraryBridgeExistsHandler"))
ExternalDictionaryLibraryBridgeExistsHandler::ExternalDictionaryLibraryBridgeExistsHandler(ContextPtr context_)
: WithContext(context_), log(getLogger("ExternalDictionaryLibraryBridgeExistsHandler"))
{
}
@ -401,7 +399,7 @@ void ExternalDictionaryLibraryBridgeExistsHandler::handleRequest(HTTPServerReque
String res = library_handler ? "1" : "0";
setResponseDefaultHeaders(response, keep_alive_timeout);
setResponseDefaultHeaders(response);
LOG_TRACE(log, "Sending ping response: {} (dictionary id: {})", res, dictionary_id);
response.sendBuffer(res.data(), res.size());
}
@ -617,10 +615,8 @@ void CatBoostLibraryBridgeRequestHandler::handleRequest(HTTPServerRequest & requ
}
CatBoostLibraryBridgeExistsHandler::CatBoostLibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_)
: WithContext(context_)
, keep_alive_timeout(keep_alive_timeout_)
, log(getLogger("CatBoostLibraryBridgeExistsHandler"))
CatBoostLibraryBridgeExistsHandler::CatBoostLibraryBridgeExistsHandler(ContextPtr context_)
: WithContext(context_), log(getLogger("CatBoostLibraryBridgeExistsHandler"))
{
}
@ -634,7 +630,7 @@ void CatBoostLibraryBridgeExistsHandler::handleRequest(HTTPServerRequest & reque
String res = "1";
setResponseDefaultHeaders(response, keep_alive_timeout);
setResponseDefaultHeaders(response);
LOG_TRACE(log, "Sending ping response: {}", res);
response.sendBuffer(res.data(), res.size());
}

6
programs/library-bridge/LibraryBridgeHandlers.h
View File

@ -34,12 +34,11 @@ private:
class ExternalDictionaryLibraryBridgeExistsHandler : public HTTPRequestHandler, WithContext
{
public:
ExternalDictionaryLibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_);
ExternalDictionaryLibraryBridgeExistsHandler(ContextPtr context_);
void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event) override;
private:
const size_t keep_alive_timeout;
LoggerPtr log;
};
@ -77,12 +76,11 @@ private:
class CatBoostLibraryBridgeExistsHandler : public HTTPRequestHandler, WithContext
{
public:
CatBoostLibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_);
CatBoostLibraryBridgeExistsHandler(ContextPtr context_);
void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event) override;
private:
const size_t keep_alive_timeout;
LoggerPtr log;
};

2
programs/odbc-bridge/PingHandler.cpp
View File

@ -10,7 +10,7 @@ void PingHandler::handleRequest(HTTPServerRequest & /* request */, HTTPServerRes
{
try
{
setResponseDefaultHeaders(response, keep_alive_timeout);
setResponseDefaultHeaders(response);
const char * data = "Ok.\n";
response.sendBuffer(data, strlen(data));
}

1
programs/server/Server.cpp
View File

@ -2251,6 +2251,7 @@ void Server::createServers(
Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
http_params->setTimeout(settings.http_receive_timeout);
http_params->setKeepAliveTimeout(global_context->getServerSettings().keep_alive_timeout);
http_params->setMaxKeepAliveRequests(static_cast<int>(global_context->getServerSettings().max_keep_alive_requests));
Poco::Util::AbstractConfiguration::Keys protocols;
config.keys("protocols", protocols);

1
src/Core/ServerSettings.h
View File

@ -113,6 +113,7 @@ namespace DB
M(Bool, async_load_databases, false, "Enable asynchronous loading of databases and tables to speedup server startup. Queries to not yet loaded entity will be blocked until load is finished.", 0) \
M(Bool, display_secrets_in_show_and_select, false, "Allow showing secrets in SHOW and SELECT queries via a format setting and a grant", 0) \
M(Seconds, keep_alive_timeout, DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT, "The number of seconds that ClickHouse waits for incoming requests before closing the connection.", 0) \
M(UInt64, max_keep_alive_requests, 10000, "The number of seconds that ClickHouse waits for incoming requests before closing the connection.", 0) \
M(Seconds, replicated_fetches_http_connection_timeout, 0, "HTTP connection timeout for part fetch requests. Inherited from default profile `http_connection_timeout` if not set explicitly.", 0) \
M(Seconds, replicated_fetches_http_send_timeout, 0, "HTTP send timeout for part fetch requests. Inherited from default profile `http_send_timeout` if not set explicitly.", 0) \
M(Seconds, replicated_fetches_http_receive_timeout, 0, "HTTP receive timeout for fetch part requests. Inherited from default profile `http_receive_timeout` if not set explicitly.", 0) \

15
src/IO/HTTPCommon.cpp
View File

@ -1,3 +1,4 @@
#include <cstddef>
#include <IO/HTTPCommon.h>
#include <Server/HTTP/HTTPServerResponse.h>
@ -33,14 +34,20 @@ namespace ErrorCodes
extern const int RECEIVED_ERROR_TOO_MANY_REQUESTS;
}
void setResponseDefaultHeaders(HTTPServerResponse & response, size_t keep_alive_timeout)
void setResponseDefaultHeaders(HTTPServerResponse & response)
{
if (!response.getKeepAlive())
return;
Poco::Timespan timeout(keep_alive_timeout, 0);
if (timeout.totalSeconds())
response.set("Keep-Alive", "timeout=" + std::to_string(timeout.totalSeconds()));
const size_t keep_alive_timeout = response.getSession().getKeepAliveTimeout();
const size_t keep_alive_max_requests = response.getSession().getMaxKeepAliveRequests();
if (keep_alive_timeout)
{
if (keep_alive_max_requests)
response.set("Keep-Alive", fmt::format("timeout={}, max={}", keep_alive_timeout, keep_alive_max_requests));
else
response.set("Keep-Alive", fmt::format("timeout={}", keep_alive_timeout));
}
}
HTTPSessionPtr makeHTTPSession(

2
src/IO/HTTPCommon.h
View File

@ -54,7 +54,7 @@ private:
using HTTPSessionPtr = std::shared_ptr<Poco::Net::HTTPClientSession>;
void setResponseDefaultHeaders(HTTPServerResponse & response, size_t keep_alive_timeout);
void setResponseDefaultHeaders(HTTPServerResponse & response);
/// Create session object to perform requests and set required parameters.
HTTPSessionPtr makeHTTPSession(

3
src/Server/HTTP/HTTPServer.cpp
View File

@ -13,7 +13,8 @@ HTTPServer::HTTPServer(
Poco::Net::HTTPServerParams::Ptr params,
const ProfileEvents::Event & read_event,
const ProfileEvents::Event & write_event)
: TCPServer(new HTTPServerConnectionFactory(context, params, factory_, read_event, write_event), thread_pool, socket_, params), factory(factory_)
: TCPServer(new HTTPServerConnectionFactory(context, params, factory_, read_event, write_event), thread_pool, socket_, params)
, factory(factory_)
{
}

2
src/Server/HTTP/HTTPServerResponse.h
View File

@ -245,6 +245,8 @@ public:
void attachRequest(HTTPServerRequest * request_) { request = request_; }
const Poco::Net::HTTPServerSession & getSession() const { return session; }
private:
Poco::Net::HTTPServerSession & session;
HTTPServerRequest * request = nullptr;

4
src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp
View File

@ -30,7 +30,7 @@ void WriteBufferFromHTTPServerResponse::startSendHeaders()
if (add_cors_header)
response.set("Access-Control-Allow-Origin", "*");
setResponseDefaultHeaders(response, keep_alive_timeout);
setResponseDefaultHeaders(response);
std::stringstream header; //STYLE_CHECK_ALLOW_STD_STRING_STREAM
response.beginWrite(header);
@ -119,12 +119,10 @@ void WriteBufferFromHTTPServerResponse::nextImpl()
WriteBufferFromHTTPServerResponse::WriteBufferFromHTTPServerResponse(
HTTPServerResponse & response_,
bool is_http_method_head_,
UInt64 keep_alive_timeout_,
const ProfileEvents::Event & write_event_)
: HTTPWriteBuffer(response_.getSocket(), write_event_)
, response(response_)
, is_http_method_head(is_http_method_head_)
, keep_alive_timeout(keep_alive_timeout_)
{
}

2
src/Server/HTTP/WriteBufferFromHTTPServerResponse.h
View File

@ -29,7 +29,6 @@ public:
WriteBufferFromHTTPServerResponse(
HTTPServerResponse & response_,
bool is_http_method_head_,
UInt64 keep_alive_timeout_,
const ProfileEvents::Event & write_event_ = ProfileEvents::end());
~WriteBufferFromHTTPServerResponse() override;
@ -91,7 +90,6 @@ private:
bool is_http_method_head;
bool add_cors_header = false;
size_t keep_alive_timeout = 0;
bool initialized = false;

3
src/Server/HTTPHandler.cpp
View File

@ -621,7 +621,6 @@ void HTTPHandler::processQuery(
std::make_shared<WriteBufferFromHTTPServerResponse>(
response,
request.getMethod() == HTTPRequest::HTTP_HEAD,
context->getServerSettings().keep_alive_timeout.totalSeconds(),
write_event);
used_output.out = used_output.out_holder;
used_output.out_maybe_compressed = used_output.out_holder;
@ -926,7 +925,7 @@ try
if (!used_output.out_holder && !used_output.exception_is_written)
{
/// If nothing was sent yet and we don't even know if we must compress the response.
WriteBufferFromHTTPServerResponse(response, request.getMethod() == HTTPRequest::HTTP_HEAD, DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT).writeln(s);
WriteBufferFromHTTPServerResponse(response, request.getMethod() == HTTPRequest::HTTP_HEAD).writeln(s);
}
else if (used_output.out_maybe_compressed)
{

3
src/Server/InterserverIOHTTPHandler.cpp
View File

@ -87,9 +87,8 @@ void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPSe
response.setChunkedTransferEncoding(true);
Output used_output;
const auto keep_alive_timeout = server.context()->getServerSettings().keep_alive_timeout.totalSeconds();
used_output.out = std::make_shared<WriteBufferFromHTTPServerResponse>(
response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout, write_event);
response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, write_event);
auto finalize_output = [&]
{

10
src/Server/PrometheusRequestHandler.cpp
View File

@ -18,21 +18,15 @@ void PrometheusRequestHandler::handleRequest(HTTPServerRequest & request, HTTPSe
{
try
{
/// Raw config reference is used here to avoid dependency on Context and ServerSettings.
/// This is painful, because this class is also used in a build with CLICKHOUSE_KEEPER_STANDALONE_BUILD=1
/// And there ordinary Context is replaced with a tiny clone.
const auto & config = server.config();
unsigned keep_alive_timeout = config.getUInt("keep_alive_timeout", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT);
/// In order to make keep-alive works.
if (request.getVersion() == HTTPServerRequest::HTTP_1_1)
response.setChunkedTransferEncoding(true);
setResponseDefaultHeaders(response, keep_alive_timeout);
setResponseDefaultHeaders(response);
response.setContentType("text/plain; version=0.0.4; charset=UTF-8");
WriteBufferFromHTTPServerResponse wb(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout, write_event);
WriteBufferFromHTTPServerResponse wb(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, write_event);
try
{
metrics_writer->write(wb);

7
src/Server/PrometheusRequestHandler.h
View File

@ -12,15 +12,10 @@ class IServer;
class PrometheusRequestHandler : public HTTPRequestHandler
{
private:
IServer & server;
PrometheusMetricsWriterPtr metrics_writer;
public:
PrometheusRequestHandler(IServer & server_, PrometheusMetricsWriterPtr metrics_writer_)
: server(server_)
, metrics_writer(std::move(metrics_writer_))
{
}
PrometheusRequestHandler(IServer &, PrometheusMetricsWriterPtr metrics_writer_) : metrics_writer(std::move(metrics_writer_)) { }
void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event) override;
};

3
src/Server/ReplicasStatusHandler.cpp
View File

@ -84,8 +84,7 @@ void ReplicasStatusHandler::handleRequest(HTTPServerRequest & request, HTTPServe
}
}
const auto & server_settings = getContext()->getServerSettings();
setResponseDefaultHeaders(response, server_settings.keep_alive_timeout.totalSeconds());
setResponseDefaultHeaders(response);
if (!ok)
{

10
src/Server/StaticRequestHandler.cpp
View File

@ -33,10 +33,9 @@ namespace ErrorCodes
extern const int INVALID_CONFIG_PARAMETER;
}
static inline std::unique_ptr<WriteBuffer>
responseWriteBuffer(HTTPServerRequest & request, HTTPServerResponse & response, UInt64 keep_alive_timeout)
static inline std::unique_ptr<WriteBuffer> responseWriteBuffer(HTTPServerRequest & request, HTTPServerResponse & response)
{
auto buf = std::unique_ptr<WriteBuffer>(new WriteBufferFromHTTPServerResponse(response, request.getMethod() == HTTPRequest::HTTP_HEAD, keep_alive_timeout));
auto buf = std::unique_ptr<WriteBuffer>(new WriteBufferFromHTTPServerResponse(response, request.getMethod() == HTTPRequest::HTTP_HEAD));
/// The client can pass a HTTP header indicating supported compression method (gzip or deflate).
String http_response_compression_methods = request.get("Accept-Encoding", "");
@ -89,8 +88,7 @@ static inline void trySendExceptionToClient(
void StaticRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & /*write_event*/)
{
auto keep_alive_timeout = server.context()->getServerSettings().keep_alive_timeout.totalSeconds();
auto out = responseWriteBuffer(request, response, keep_alive_timeout);
auto out = responseWriteBuffer(request, response);
try
{
@ -105,7 +103,7 @@ void StaticRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServer
"The Transfer-Encoding is not chunked and there "
"is no Content-Length header for POST request");
setResponseDefaultHeaders(response, keep_alive_timeout);
setResponseDefaultHeaders(response);
response.setStatusAndReason(Poco::Net::HTTPResponse::HTTPStatus(status));
writeResponse(*out);
}

9
src/Server/WebUIRequestHandler.cpp
View File

@ -29,18 +29,15 @@ DashboardWebUIRequestHandler::DashboardWebUIRequestHandler(IServer & server_) :
BinaryWebUIRequestHandler::BinaryWebUIRequestHandler(IServer & server_) : server(server_) {}
JavaScriptWebUIRequestHandler::JavaScriptWebUIRequestHandler(IServer & server_) : server(server_) {}
static void handle(const IServer & server, HTTPServerRequest & request, HTTPServerResponse & response, std::string_view html)
static void handle(const IServer &, HTTPServerRequest & request, HTTPServerResponse & response, std::string_view html)
{
auto keep_alive_timeout = server.context()->getServerSettings().keep_alive_timeout.totalSeconds();
response.setContentType("text/html; charset=UTF-8");
if (request.getVersion() == HTTPServerRequest::HTTP_1_1)
response.setChunkedTransferEncoding(true);
setResponseDefaultHeaders(response, keep_alive_timeout);
setResponseDefaultHeaders(response);
response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_OK);
WriteBufferFromHTTPServerResponse(response, request.getMethod() == HTTPRequest::HTTP_HEAD, keep_alive_timeout).write(html.data(), html.size());
WriteBufferFromHTTPServerResponse(response, request.getMethod() == HTTPRequest::HTTP_HEAD).write(html.data(), html.size());
}
void PlayWebUIRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event &)

6
tests/queries/0_stateless/00408_http_keep_alive.reference
View File

@ -1,6 +1,6 @@
< Connection: Keep-Alive
< Keep-Alive: timeout=10
< Keep-Alive: timeout=10, max=10000
< Connection: Keep-Alive
< Keep-Alive: timeout=10
< Keep-Alive: timeout=10, max=10000
< Connection: Keep-Alive
< Keep-Alive: timeout=10
< Keep-Alive: timeout=10, max=10000

4
tests/queries/0_stateless/00501_http_head.reference
View File

@ -2,11 +2,11 @@ HTTP/1.1 200 OK
Connection: Keep-Alive
Content-Type: text/tab-separated-values; charset=UTF-8
Transfer-Encoding: chunked
Keep-Alive: timeout=10
Keep-Alive: timeout=10, max=10000
HTTP/1.1 200 OK
Connection: Keep-Alive
Content-Type: text/tab-separated-values; charset=UTF-8
Transfer-Encoding: chunked
Keep-Alive: timeout=10
Keep-Alive: timeout=10, max=10000