thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #22323 from abyss7/poco-http2

Browse Source 
Follow up to #19516
This commit is contained in:
alexey-milovidov 2021-04-03 02:09:10 +03:00 committed by GitHub
commit 9f5cd35a69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 33 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/Core/Settings.h
View File

@ -228,6 +228,7 @@ class IColumn;
M(Seconds, http_connection_timeout, DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT, "HTTP connection timeout.", 0) \
M(Seconds, http_send_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP send timeout", 0) \
M(Seconds, http_receive_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP receive timeout", 0) \
M(UInt64, http_max_uri_size, 16384, "Maximum URI length of HTTP request", 0) \
M(Bool, optimize_throw_if_noop, false, "If setting is enabled and OPTIMIZE query didn't actually assign a merge then an explanatory exception is thrown", 0) \
M(Bool, use_index_for_in_with_subqueries, true, "Try using an index if there is a subquery or a table expression on the right side of the IN operator.", 0) \
M(Bool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \
@ -546,7 +547,7 @@ struct Settings : public BaseSettings<SettingsTraits>
{
/// For initialization from empty initializer-list to be "value initialization", not "aggregate initialization" in C++14.
/// http://en.cppreference.com/w/cpp/language/aggregate_initialization
Settings() {}
Settings() = default;
/** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings).
* The profile can also be set using the `set` functions, like the profile setting.

57
src/Server/HTTP/HTMLForm.cpp
View File

@ -71,23 +71,6 @@ HTMLForm::HTMLForm(const Poco::URI & uri) : field_limit(DFL_FIELD_LIMIT), value_
}
void HTMLForm::setEncoding(const std::string & encoding_)
{
encoding = encoding_;
}
void HTMLForm::addPart(const std::string & name, Poco::Net::PartSource * source)
{
poco_check_ptr(source);
Part part;
part.name = name;
part.source = std::unique_ptr<Poco::Net::PartSource>(source);
parts.push_back(std::move(part));
}
void HTMLForm::load(const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler)
{
clear();
@ -126,36 +109,12 @@ void HTMLForm::load(const Poco::Net::HTTPRequest & request, ReadBuffer & request
}
void HTMLForm::load(const Poco::Net::HTTPRequest & request)
{
NullPartHandler nah;
EmptyReadBuffer nis;
load(request, nis, nah);
}
void HTMLForm::read(ReadBuffer & in, PartHandler & handler)
{
if (encoding == ENCODING_URL)
readQuery(in);
else
readMultipart(in, handler);
}
void HTMLForm::read(ReadBuffer & in)
{
readQuery(in);
}
void HTMLForm::read(const std::string & queryString)
{
ReadBufferFromString istr(queryString);
readQuery(istr);
}
void HTMLForm::readQuery(ReadBuffer & in)
{
size_t fields = 0;
@ -269,22 +228,6 @@ void HTMLForm::readMultipart(ReadBuffer & in_, PartHandler & handler)
}
void HTMLForm::setFieldLimit(int limit)
{
poco_assert(limit >= 0);
field_limit = limit;
}
void HTMLForm::setValueLengthLimit(int limit)
{
poco_assert(limit >= 0);
value_length_limit = limit;
}
HTMLForm::MultipartReadBuffer::MultipartReadBuffer(ReadBuffer & in_, const std::string & boundary_)
: ReadBuffer(nullptr, 0), in(in_), boundary("--" + boundary_)
{

49
src/Server/HTTP/HTMLForm.h
View File

@ -52,24 +52,6 @@ public:
return (it != end()) ? DB::parse<T>(it->second) : default_value;
}
template <typename T>
T getParsed(const std::string & key)
{
return DB::parse<T>(get(key));
}
/// Sets the encoding used for posting the form.
/// Encoding must be either "application/x-www-form-urlencoded" (which is the default) or "multipart/form-data".
void setEncoding(const std::string & encoding);
/// Returns the encoding used for posting the form.
const std::string & getEncoding() const { return encoding; }
/// Adds an part/attachment (file upload) to the form.
/// The form takes ownership of the PartSource and deletes it when it is no longer needed.
/// The part will only be sent if the encoding set for the form is "multipart/form-data"
void addPart(const std::string & name, Poco::Net::PartSource * pSource);
/// Reads the form data from the given HTTP request.
/// Uploaded files are passed to the given PartHandler.
void load(const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler);
@ -78,41 +60,10 @@ public:
/// Uploaded files are silently discarded.
void load(const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody);
/// Reads the form data from the given HTTP request.
/// The request must be a GET request and the form data must be in the query string (URL encoded).
/// For POST requests, you must use one of the overloads taking an additional input stream for the request body.
void load(const Poco::Net::HTTPRequest & request);
/// Reads the form data from the given input stream.
/// The form data read from the stream must be in the encoding specified for the form.
/// Note that read() does not clear the form before reading the new values.
void read(ReadBuffer & in, PartHandler & handler);
/// Reads the URL-encoded form data from the given input stream.
/// Note that read() does not clear the form before reading the new values.
void read(ReadBuffer & in);
/// Reads the form data from the given HTTP query string.
/// Note that read() does not clear the form before reading the new values.
void read(const std::string & queryString);
/// Returns the MIME boundary used for writing multipart form data.
const std::string & getBoundary() const { return boundary; }
/// Returns the maximum number of header fields allowed.
/// See setFieldLimit() for more information.
int getFieldLimit() const { return field_limit; }
/// Sets the maximum number of header fields allowed. This limit is used to defend certain kinds of denial-of-service attacks.
/// Specify 0 for unlimited (not recommended). The default limit is 100.
void setFieldLimit(int limit);
/// Sets the maximum size for form field values stored as strings.
void setValueLengthLimit(int limit);
/// Returns the maximum size for form field values stored as strings.
int getValueLengthLimit() const { return value_length_limit; }
static const std::string ENCODING_URL; /// "application/x-www-form-urlencoded"
static const std::string ENCODING_MULTIPART; /// "multipart/form-data"
static const int UNKNOWN_CONTENT_LENGTH;

8
src/Server/HTTP/HTTPServer.cpp
View File

@ -8,9 +8,9 @@ namespace DB
HTTPServer::HTTPServer(
const Context & context,
HTTPRequestHandlerFactoryPtr factory_,
UInt16 portNumber,
UInt16 port_number,
Poco::Net::HTTPServerParams::Ptr params)
: TCPServer(new HTTPServerConnectionFactory(context, params, factory_), portNumber, params), factory(factory_)
: TCPServer(new HTTPServerConnectionFactory(context, params, factory_), port_number, params), factory(factory_)
{
}
@ -26,10 +26,10 @@ HTTPServer::HTTPServer(
HTTPServer::HTTPServer(
const Context & context,
HTTPRequestHandlerFactoryPtr factory_,
Poco::ThreadPool & threadPool,
Poco::ThreadPool & thread_pool,
const Poco::Net::ServerSocket & socket,
Poco::Net::HTTPServerParams::Ptr params)
: TCPServer(new HTTPServerConnectionFactory(context, params, factory_), threadPool, socket, params), factory(factory_)
: TCPServer(new HTTPServerConnectionFactory(context, params, factory_), thread_pool, socket, params), factory(factory_)
{
}

6
src/Server/HTTP/HTTPServer.h
View File

@ -19,7 +19,7 @@ public:
explicit HTTPServer(
const Context & context,
HTTPRequestHandlerFactoryPtr factory,
UInt16 portNumber = 80,
UInt16 port_number = 80,
Poco::Net::HTTPServerParams::Ptr params = new Poco::Net::HTTPServerParams);
HTTPServer(
@ -31,13 +31,13 @@ public:
HTTPServer(
const Context & context,
HTTPRequestHandlerFactoryPtr factory,
Poco::ThreadPool & threadPool,
Poco::ThreadPool & thread_pool,
const Poco::Net::ServerSocket & socket,
Poco::Net::HTTPServerParams::Ptr params);
~HTTPServer() override;
void stopAll(bool abortCurrent = false);
void stopAll(bool abort_current = false);
private:
HTTPRequestHandlerFactoryPtr factory;

33
src/Server/HTTP/HTTPServerConnection.cpp
View File

@ -67,15 +67,15 @@ void HTTPServerConnection::run()
}
}
}
catch (Poco::Net::NoMessageException &)
catch (const Poco::Net::NoMessageException &)
{
break;
}
catch (Poco::Net::MessageException &)
catch (const Poco::Net::MessageException &)
{
sendErrorResponse(session, Poco::Net::HTTPResponse::HTTP_BAD_REQUEST);
}
catch (Poco::Exception &)
catch (const Poco::Exception &)
{
if (session.networkException())
{
@ -98,31 +98,4 @@ void HTTPServerConnection::sendErrorResponse(Poco::Net::HTTPServerSession & sess
session.setKeepAlive(false);
}
void HTTPServerConnection::onServerStopped(const bool & abortCurrent)
{
stopped = true;
if (abortCurrent)
{
try
{
socket().shutdown();
}
catch (...)
{
}
}
else
{
std::unique_lock<std::mutex> lock(mutex);
try
{
socket().shutdown();
}
catch (...)
{
}
}
}
}

1
src/Server/HTTP/HTTPServerConnection.h
View File

@ -23,7 +23,6 @@ public:
protected:
static void sendErrorResponse(Poco::Net::HTTPServerSession & session, Poco::Net::HTTPResponse::HTTPStatus status);
void onServerStopped(const bool & abortCurrent);
private:
Context context;

6
src/Server/HTTP/HTTPServerRequest.cpp
View File

@ -15,8 +15,8 @@
namespace DB
{
HTTPServerRequest::HTTPServerRequest(const Context & context, HTTPServerResponse & response, Poco::Net::HTTPServerSession & session)
: max_uri_size(context.getSettingsRef().http_max_uri_size)
{
response.attachRequest(this);
@ -92,10 +92,10 @@ void HTTPServerRequest::readRequest(ReadBuffer & in)
skipWhitespaceIfAny(in);
while (in.read(ch) && !Poco::Ascii::isSpace(ch) && uri.size() <= MAX_URI_LENGTH)
while (in.read(ch) && !Poco::Ascii::isSpace(ch) && uri.size() <= max_uri_size)
uri += ch;
if (uri.size() > MAX_URI_LENGTH)
if (uri.size() > max_uri_size)
throw Poco::Net::MessageException("HTTP request URI invalid or too long");
skipWhitespaceIfAny(in);

3
src/Server/HTTP/HTTPServerRequest.h
View File

@ -43,11 +43,12 @@ private:
MAX_NAME_LENGTH = 256,
MAX_VALUE_LENGTH = 8192,
MAX_METHOD_LENGTH = 32,
MAX_URI_LENGTH = 16384,
MAX_VERSION_LENGTH = 8,
MAX_FIELDS_NUMBER = 100,
};
const size_t max_uri_size;
std::unique_ptr<ReadBuffer> stream;
Poco::Net::SocketImpl * socket;
Poco::Net::SocketAddress client_address;

40
src/Server/HTTP/HTTPServerResponse.cpp
View File

@ -94,32 +94,6 @@ std::pair<std::shared_ptr<std::ostream>, std::shared_ptr<std::ostream>> HTTPServ
return std::make_pair(header_stream, stream);
}
void HTTPServerResponse::sendFile(const std::string & path, const std::string & mediaType)
{
poco_assert(!stream);
Poco::File f(path);
Poco::Timestamp date_time = f.getLastModified();
Poco::File::FileSize length = f.getSize();
set("Last-Modified", Poco::DateTimeFormatter::format(date_time, Poco::DateTimeFormat::HTTP_FORMAT));
setContentLength64(length);
setContentType(mediaType);
setChunkedTransferEncoding(false);
Poco::FileInputStream istr(path);
if (istr.good())
{
stream = std::make_shared<Poco::Net::HTTPHeaderOutputStream>(session);
write(*stream);
if (request && request->getMethod() != HTTPRequest::HTTP_HEAD)
{
Poco::StreamCopier::copyStream(istr, *stream);
}
}
else
throw Poco::OpenFileException(path);
}
void HTTPServerResponse::sendBuffer(const void * buffer, std::size_t length)
{
poco_assert(!stream);
@ -135,20 +109,6 @@ void HTTPServerResponse::sendBuffer(const void * buffer, std::size_t length)
}
}
void HTTPServerResponse::redirect(const std::string & uri, HTTPStatus status)
{
poco_assert(!stream);
setContentLength(0);
setChunkedTransferEncoding(false);
setStatusAndReason(status);
set("Location", uri);
stream = std::make_shared<Poco::Net::HTTPHeaderOutputStream>(session);
write(*stream);
}
void HTTPServerResponse::requireAuthentication(const std::string & realm)
{
poco_assert(!stream);

23
src/Server/HTTP/HTTPServerResponse.h
View File

@ -36,17 +36,6 @@ public:
/// or redirect() has been called.
std::pair<std::shared_ptr<std::ostream>, std::shared_ptr<std::ostream>> beginSend(); /// TODO: use some WriteBuffer implementation here.
/// Sends the response header to the client, followed
/// by the content of the given file.
///
/// Must not be called after send(), sendBuffer()
/// or redirect() has been called.
///
/// Throws a FileNotFoundException if the file
/// cannot be found, or an OpenFileException if
/// the file cannot be opened.
void sendFile(const std::string & path, const std::string & mediaType);
/// Sends the response header to the client, followed
/// by the contents of the given buffer.
///
@ -61,16 +50,6 @@ public:
/// or redirect() has been called.
void sendBuffer(const void * pBuffer, std::size_t length); /// FIXME: do we need this one?
/// Sets the status code, which must be one of
/// HTTP_MOVED_PERMANENTLY (301), HTTP_FOUND (302),
/// or HTTP_SEE_OTHER (303),
/// and sets the "Location" header field
/// to the given URI, which according to
/// the HTTP specification, must be absolute.
///
/// Must not be called after send() has been called.
void redirect(const std::string & uri, Poco::Net::HTTPResponse::HTTPStatus status = Poco::Net::HTTPResponse::HTTP_FOUND);
void requireAuthentication(const std::string & realm);
/// Sets the status code to 401 (Unauthorized)
/// and sets the "WWW-Authenticate" header field
@ -83,7 +62,7 @@ public:
private:
Poco::Net::HTTPServerSession & session;
HTTPServerRequest * request;
HTTPServerRequest * request = nullptr;
std::shared_ptr<std::ostream> stream;
std::shared_ptr<std::ostream> header_stream;
};

3
src/Server/HTTP/README.md Normal file
View File

@ -0,0 +1,3 @@
# Notice
The source code located in this folder is based on some files from the POCO project, from here `contrib/poco/Net/src`.

1
tests/queries/0_stateless/01753_max_uri_size.reference Normal file
View File

@ -0,0 +1 @@
1

11
tests/queries/0_stateless/01753_max_uri_size.sh Executable file
View File

@ -0,0 +1,11 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
# NOTE: since 'max_uri_size' doesn't affect the request itself, this test hardly depends on the default value of this setting (16Kb).
LONG_REQUEST=$(python3 -c "print('&max_uri_size=1'*2000, end='')") # ~30K
${CLICKHOUSE_CURL} -sSv "${CLICKHOUSE_URL}${LONG_REQUEST}&query=SELECT+1" 2>&1 | grep -Fc "HTTP/1.1 400 Bad Request"