[feature] A setting http_response_headers

Implementing https://github.com/ClickHouse/ClickHouse/issues/59620
Deprecating `content_type` setting (still supported).
This commit is contained in:
Grigorii Sokolik 2024-05-09 14:27:43 +03:00
parent 72cc6b2ba0
commit 47686e0c4a
2 changed files with 308 additions and 282 deletions

View File

@ -1,8 +1,8 @@
#include <Server/HTTPHandler.h>
#include <Access/AccessControl.h>
#include <Access/Authentication.h>
#include <Access/Credentials.h>
#include <Access/AccessControl.h>
#include <Access/ExternalAuthenticators.h>
#include <Access/Role.h>
#include <Access/User.h>
@ -10,6 +10,7 @@
#include <Compression/CompressedWriteBuffer.h>
#include <Core/ExternalTable.h>
#include <Disks/StoragePolicy.h>
#include <Formats/FormatFactory.h>
#include <IO/CascadeWriteBuffer.h>
#include <IO/ConcatReadBuffer.h>
#include <IO/MemoryReadWriteBuffer.h>
@ -17,46 +18,47 @@
#include <IO/WriteHelpers.h>
#include <IO/copyData.h>
#include <Interpreters/Context.h>
#include <Interpreters/TemporaryDataOnDisk.h>
#include <Parsers/QueryParameterVisitor.h>
#include <Interpreters/executeQuery.h>
#include <Interpreters/Session.h>
#include <Interpreters/TemporaryDataOnDisk.h>
#include <Interpreters/executeQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Parsers/QueryParameterVisitor.h>
#include <Processors/Formats/IOutputFormat.h>
#include <Server/HTTPHandlerFactory.h>
#include <Server/HTTPHandlerRequestFilter.h>
#include <Server/IServer.h>
#include <Common/logger_useful.h>
#include <Common/SettingsChanges.h>
#include <Common/StringUtils.h>
#include <Common/scope_guard_safe.h>
#include <Common/setThreadName.h>
#include <Common/typeid_cast.h>
#include <Common/re2.h>
#include <Parsers/ASTSetQuery.h>
#include <Processors/Formats/IOutputFormat.h>
#include <Formats/FormatFactory.h>
#include <Server/HTTP/HTTPResponse.h>
#include <base/getFQDNOrHostName.h>
#include <base/scope_guard.h>
#include <Server/HTTP/HTTPResponse.h>
#include "config.h"
#include <Poco/Base64Decoder.h>
#include <Poco/Base64Encoder.h>
#include <Poco/Net/HTTPBasicCredentials.h>
#include <Poco/Net/HTTPStream.h>
#include <Poco/MemoryStream.h>
#include <Poco/Net/HTTPBasicCredentials.h>
#include <Poco/Net/HTTPMessage.h>
#include <Poco/Net/HTTPStream.h>
#include <Poco/Net/SocketAddress.h>
#include <Poco/StreamCopier.h>
#include <Poco/String.h>
#include <Poco/Net/SocketAddress.h>
#include <algorithm>
#include <chrono>
#include <memory>
#include <optional>
#include <sstream>
#include <unordered_map>
#include <utility>
#if USE_SSL
#include <Poco/Net/X509Certificate.h>
# include <Poco/Net/X509Certificate.h>
#endif
@ -65,68 +67,68 @@ namespace DB
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
extern const int CANNOT_COMPILE_REGEXP;
extern const int CANNOT_OPEN_FILE;
extern const int CANNOT_PARSE_TEXT;
extern const int CANNOT_PARSE_ESCAPE_SEQUENCE;
extern const int CANNOT_PARSE_QUOTED_STRING;
extern const int CANNOT_PARSE_DATE;
extern const int CANNOT_PARSE_DATETIME;
extern const int CANNOT_PARSE_NUMBER;
extern const int CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING;
extern const int CANNOT_PARSE_IPV4;
extern const int CANNOT_PARSE_IPV6;
extern const int CANNOT_PARSE_UUID;
extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED;
extern const int CANNOT_SCHEDULE_TASK;
extern const int DUPLICATE_COLUMN;
extern const int ILLEGAL_COLUMN;
extern const int THERE_IS_NO_COLUMN;
extern const int UNKNOWN_ELEMENT_IN_AST;
extern const int UNKNOWN_TYPE_OF_AST_NODE;
extern const int TOO_DEEP_AST;
extern const int TOO_BIG_AST;
extern const int UNEXPECTED_AST_STRUCTURE;
extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE;
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
extern const int CANNOT_SCHEDULE_TASK;
extern const int CANNOT_PARSE_TEXT;
extern const int CANNOT_PARSE_ESCAPE_SEQUENCE;
extern const int CANNOT_PARSE_QUOTED_STRING;
extern const int CANNOT_PARSE_DATE;
extern const int CANNOT_PARSE_DATETIME;
extern const int CANNOT_PARSE_NUMBER;
extern const int CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING;
extern const int CANNOT_PARSE_IPV4;
extern const int CANNOT_PARSE_IPV6;
extern const int CANNOT_PARSE_UUID;
extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED;
extern const int CANNOT_OPEN_FILE;
extern const int CANNOT_COMPILE_REGEXP;
extern const int DUPLICATE_COLUMN;
extern const int ILLEGAL_COLUMN;
extern const int THERE_IS_NO_COLUMN;
extern const int UNKNOWN_ELEMENT_IN_AST;
extern const int UNKNOWN_TYPE_OF_AST_NODE;
extern const int TOO_DEEP_AST;
extern const int TOO_BIG_AST;
extern const int UNEXPECTED_AST_STRUCTURE;
extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE;
extern const int SYNTAX_ERROR;
extern const int SYNTAX_ERROR;
extern const int INCORRECT_DATA;
extern const int TYPE_MISMATCH;
extern const int INCORRECT_DATA;
extern const int TYPE_MISMATCH;
extern const int UNKNOWN_TABLE;
extern const int UNKNOWN_FUNCTION;
extern const int UNKNOWN_IDENTIFIER;
extern const int UNKNOWN_TYPE;
extern const int UNKNOWN_STORAGE;
extern const int UNKNOWN_DATABASE;
extern const int UNKNOWN_SETTING;
extern const int UNKNOWN_DIRECTION_OF_SORTING;
extern const int UNKNOWN_AGGREGATE_FUNCTION;
extern const int UNKNOWN_FORMAT;
extern const int UNKNOWN_DATABASE_ENGINE;
extern const int UNKNOWN_TYPE_OF_QUERY;
extern const int UNKNOWN_ROLE;
extern const int NO_ELEMENTS_IN_CONFIG;
extern const int UNKNOWN_TABLE;
extern const int UNKNOWN_FUNCTION;
extern const int UNKNOWN_IDENTIFIER;
extern const int UNKNOWN_TYPE;
extern const int UNKNOWN_STORAGE;
extern const int UNKNOWN_DATABASE;
extern const int UNKNOWN_SETTING;
extern const int UNKNOWN_DIRECTION_OF_SORTING;
extern const int UNKNOWN_AGGREGATE_FUNCTION;
extern const int UNKNOWN_FORMAT;
extern const int UNKNOWN_DATABASE_ENGINE;
extern const int UNKNOWN_TYPE_OF_QUERY;
extern const int UNKNOWN_ROLE;
extern const int NO_ELEMENTS_IN_CONFIG;
extern const int QUERY_IS_TOO_LARGE;
extern const int QUERY_IS_TOO_LARGE;
extern const int NOT_IMPLEMENTED;
extern const int SOCKET_TIMEOUT;
extern const int NOT_IMPLEMENTED;
extern const int SOCKET_TIMEOUT;
extern const int UNKNOWN_USER;
extern const int WRONG_PASSWORD;
extern const int REQUIRED_PASSWORD;
extern const int AUTHENTICATION_FAILED;
extern const int SET_NON_GRANTED_ROLE;
extern const int UNKNOWN_USER;
extern const int WRONG_PASSWORD;
extern const int REQUIRED_PASSWORD;
extern const int AUTHENTICATION_FAILED;
extern const int SET_NON_GRANTED_ROLE;
extern const int INVALID_SESSION_TIMEOUT;
extern const int HTTP_LENGTH_REQUIRED;
extern const int SUPPORT_IS_DISABLED;
extern const int INVALID_SESSION_TIMEOUT;
extern const int HTTP_LENGTH_REQUIRED;
extern const int SUPPORT_IS_DISABLED;
extern const int TIMEOUT_EXCEEDED;
extern const int TIMEOUT_EXCEEDED;
}
namespace
@ -145,9 +147,9 @@ bool tryAddHTTPOptionHeadersFromConfig(HTTPServerResponse & response, const Poco
if (config.getString("http_options_response." + config_key + ".name", "").empty())
LOG_WARNING(getLogger("processOptionsRequest"), "Empty header was found in config. It will not be processed.");
else
response.add(config.getString("http_options_response." + config_key + ".name", ""),
config.getString("http_options_response." + config_key + ".value", ""));
response.add(
config.getString("http_options_response." + config_key + ".name", ""),
config.getString("http_options_response." + config_key + ".value", ""));
}
}
return true;
@ -196,54 +198,37 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti
{
return HTTPResponse::HTTP_UNAUTHORIZED;
}
else if (exception_code == ErrorCodes::UNKNOWN_USER ||
exception_code == ErrorCodes::WRONG_PASSWORD ||
exception_code == ErrorCodes::AUTHENTICATION_FAILED ||
exception_code == ErrorCodes::SET_NON_GRANTED_ROLE)
else if (
exception_code == ErrorCodes::UNKNOWN_USER || exception_code == ErrorCodes::WRONG_PASSWORD
|| exception_code == ErrorCodes::AUTHENTICATION_FAILED || exception_code == ErrorCodes::SET_NON_GRANTED_ROLE)
{
return HTTPResponse::HTTP_FORBIDDEN;
}
else if (exception_code == ErrorCodes::BAD_ARGUMENTS ||
exception_code == ErrorCodes::CANNOT_COMPILE_REGEXP ||
exception_code == ErrorCodes::CANNOT_PARSE_TEXT ||
exception_code == ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE ||
exception_code == ErrorCodes::CANNOT_PARSE_QUOTED_STRING ||
exception_code == ErrorCodes::CANNOT_PARSE_DATE ||
exception_code == ErrorCodes::CANNOT_PARSE_DATETIME ||
exception_code == ErrorCodes::CANNOT_PARSE_NUMBER ||
exception_code == ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING ||
exception_code == ErrorCodes::CANNOT_PARSE_IPV4 ||
exception_code == ErrorCodes::CANNOT_PARSE_IPV6 ||
exception_code == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED ||
exception_code == ErrorCodes::CANNOT_PARSE_UUID ||
exception_code == ErrorCodes::DUPLICATE_COLUMN ||
exception_code == ErrorCodes::ILLEGAL_COLUMN ||
exception_code == ErrorCodes::UNKNOWN_ELEMENT_IN_AST ||
exception_code == ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE ||
exception_code == ErrorCodes::THERE_IS_NO_COLUMN ||
exception_code == ErrorCodes::TOO_DEEP_AST ||
exception_code == ErrorCodes::TOO_BIG_AST ||
exception_code == ErrorCodes::UNEXPECTED_AST_STRUCTURE ||
exception_code == ErrorCodes::SYNTAX_ERROR ||
exception_code == ErrorCodes::INCORRECT_DATA ||
exception_code == ErrorCodes::TYPE_MISMATCH ||
exception_code == ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE)
else if (
exception_code == ErrorCodes::BAD_ARGUMENTS || exception_code == ErrorCodes::CANNOT_COMPILE_REGEXP
|| exception_code == ErrorCodes::CANNOT_PARSE_TEXT || exception_code == ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE
|| exception_code == ErrorCodes::CANNOT_PARSE_QUOTED_STRING || exception_code == ErrorCodes::CANNOT_PARSE_DATE
|| exception_code == ErrorCodes::CANNOT_PARSE_DATETIME || exception_code == ErrorCodes::CANNOT_PARSE_NUMBER
|| exception_code == ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING || exception_code == ErrorCodes::CANNOT_PARSE_IPV4
|| exception_code == ErrorCodes::CANNOT_PARSE_IPV6 || exception_code == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED
|| exception_code == ErrorCodes::CANNOT_PARSE_UUID || exception_code == ErrorCodes::DUPLICATE_COLUMN
|| exception_code == ErrorCodes::ILLEGAL_COLUMN || exception_code == ErrorCodes::UNKNOWN_ELEMENT_IN_AST
|| exception_code == ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE || exception_code == ErrorCodes::THERE_IS_NO_COLUMN
|| exception_code == ErrorCodes::TOO_DEEP_AST || exception_code == ErrorCodes::TOO_BIG_AST
|| exception_code == ErrorCodes::UNEXPECTED_AST_STRUCTURE || exception_code == ErrorCodes::SYNTAX_ERROR
|| exception_code == ErrorCodes::INCORRECT_DATA || exception_code == ErrorCodes::TYPE_MISMATCH
|| exception_code == ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE)
{
return HTTPResponse::HTTP_BAD_REQUEST;
}
else if (exception_code == ErrorCodes::UNKNOWN_TABLE ||
exception_code == ErrorCodes::UNKNOWN_FUNCTION ||
exception_code == ErrorCodes::UNKNOWN_IDENTIFIER ||
exception_code == ErrorCodes::UNKNOWN_TYPE ||
exception_code == ErrorCodes::UNKNOWN_STORAGE ||
exception_code == ErrorCodes::UNKNOWN_DATABASE ||
exception_code == ErrorCodes::UNKNOWN_SETTING ||
exception_code == ErrorCodes::UNKNOWN_DIRECTION_OF_SORTING ||
exception_code == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION ||
exception_code == ErrorCodes::UNKNOWN_FORMAT ||
exception_code == ErrorCodes::UNKNOWN_DATABASE_ENGINE ||
exception_code == ErrorCodes::UNKNOWN_TYPE_OF_QUERY ||
exception_code == ErrorCodes::UNKNOWN_ROLE)
else if (
exception_code == ErrorCodes::UNKNOWN_TABLE || exception_code == ErrorCodes::UNKNOWN_FUNCTION
|| exception_code == ErrorCodes::UNKNOWN_IDENTIFIER || exception_code == ErrorCodes::UNKNOWN_TYPE
|| exception_code == ErrorCodes::UNKNOWN_STORAGE || exception_code == ErrorCodes::UNKNOWN_DATABASE
|| exception_code == ErrorCodes::UNKNOWN_SETTING || exception_code == ErrorCodes::UNKNOWN_DIRECTION_OF_SORTING
|| exception_code == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION || exception_code == ErrorCodes::UNKNOWN_FORMAT
|| exception_code == ErrorCodes::UNKNOWN_DATABASE_ENGINE || exception_code == ErrorCodes::UNKNOWN_TYPE_OF_QUERY
|| exception_code == ErrorCodes::UNKNOWN_ROLE)
{
return HTTPResponse::HTTP_NOT_FOUND;
}
@ -255,8 +240,7 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti
{
return HTTPResponse::HTTP_NOT_IMPLEMENTED;
}
else if (exception_code == ErrorCodes::SOCKET_TIMEOUT ||
exception_code == ErrorCodes::CANNOT_OPEN_FILE)
else if (exception_code == ErrorCodes::SOCKET_TIMEOUT || exception_code == ErrorCodes::CANNOT_OPEN_FILE)
{
return HTTPResponse::HTTP_SERVICE_UNAVAILABLE;
}
@ -277,9 +261,7 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti
}
static std::chrono::steady_clock::duration parseSessionTimeout(
const Poco::Util::AbstractConfiguration & config,
const HTMLForm & params)
static std::chrono::steady_clock::duration parseSessionTimeout(const Poco::Util::AbstractConfiguration & config, const HTMLForm & params)
{
unsigned session_timeout = config.getInt("default_session_timeout", 60);
@ -293,14 +275,19 @@ static std::chrono::steady_clock::duration parseSessionTimeout(
throw Exception(ErrorCodes::INVALID_SESSION_TIMEOUT, "Invalid session timeout: '{}'", session_timeout_str);
if (session_timeout > max_session_timeout)
throw Exception(ErrorCodes::INVALID_SESSION_TIMEOUT, "Session timeout '{}' is larger than max_session_timeout: {}. "
throw Exception(
ErrorCodes::INVALID_SESSION_TIMEOUT,
"Session timeout '{}' is larger than max_session_timeout: {}. "
"Maximum session timeout could be modified in configuration file.",
session_timeout_str, max_session_timeout);
session_timeout_str,
max_session_timeout);
}
return std::chrono::seconds(session_timeout);
}
std::optional<std::unordered_map<String, String>>
parseHttpResponseHeaders(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix);
void HTTPHandler::pushDelayedResults(Output & used_output)
{
@ -338,11 +325,12 @@ void HTTPHandler::pushDelayedResults(Output & used_output)
}
HTTPHandler::HTTPHandler(IServer & server_, const std::string & name, const std::optional<String> & content_type_override_)
HTTPHandler::HTTPHandler(
IServer & server_, const std::string & name, const std::optional<std::unordered_map<String, String>> & http_response_headers_override_)
: server(server_)
, log(getLogger(name))
, default_settings(server.context()->getSettingsRef())
, content_type_override(content_type_override_)
, http_response_headers_override(http_response_headers_override_)
{
server_display_name = server.config().getString("display_name", getFQDNOrHostName());
}
@ -353,10 +341,7 @@ HTTPHandler::HTTPHandler(IServer & server_, const std::string & name, const std:
HTTPHandler::~HTTPHandler() = default;
bool HTTPHandler::authenticateUser(
HTTPServerRequest & request,
HTMLForm & params,
HTTPServerResponse & response)
bool HTTPHandler::authenticateUser(HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response)
{
using namespace Poco::Net;
@ -383,31 +368,36 @@ bool HTTPHandler::authenticateUser(
{
/// It is prohibited to mix different authorization schemes.
if (has_http_credentials)
throw Exception(ErrorCodes::AUTHENTICATION_FAILED,
"Invalid authentication: it is not allowed "
"to use SSL certificate authentication and Authorization HTTP header simultaneously");
throw Exception(
ErrorCodes::AUTHENTICATION_FAILED,
"Invalid authentication: it is not allowed "
"to use SSL certificate authentication and Authorization HTTP header simultaneously");
if (has_credentials_in_query_params)
throw Exception(ErrorCodes::AUTHENTICATION_FAILED,
"Invalid authentication: it is not allowed "
"to use SSL certificate authentication and authentication via parameters simultaneously simultaneously");
throw Exception(
ErrorCodes::AUTHENTICATION_FAILED,
"Invalid authentication: it is not allowed "
"to use SSL certificate authentication and authentication via parameters simultaneously simultaneously");
if (has_ssl_certificate_auth)
{
#if USE_SSL
if (!password.empty())
throw Exception(ErrorCodes::AUTHENTICATION_FAILED,
"Invalid authentication: it is not allowed "
"to use SSL certificate authentication and authentication via password simultaneously");
throw Exception(
ErrorCodes::AUTHENTICATION_FAILED,
"Invalid authentication: it is not allowed "
"to use SSL certificate authentication and authentication via password simultaneously");
if (request.havePeerCertificate())
certificate_common_name = request.peerCertificate().commonName();
if (certificate_common_name.empty())
throw Exception(ErrorCodes::AUTHENTICATION_FAILED,
"Invalid authentication: SSL certificate authentication requires nonempty certificate's Common Name");
throw Exception(
ErrorCodes::AUTHENTICATION_FAILED,
"Invalid authentication: SSL certificate authentication requires nonempty certificate's Common Name");
#else
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
"SSL certificate authentication disabled because ClickHouse was built without SSL library");
throw Exception(
ErrorCodes::SUPPORT_IS_DISABLED,
"SSL certificate authentication disabled because ClickHouse was built without SSL library");
#endif
}
}
@ -415,9 +405,10 @@ bool HTTPHandler::authenticateUser(
{
/// It is prohibited to mix different authorization schemes.
if (has_credentials_in_query_params)
throw Exception(ErrorCodes::AUTHENTICATION_FAILED,
"Invalid authentication: it is not allowed "
"to use Authorization HTTP header and authentication via parameters simultaneously");
throw Exception(
ErrorCodes::AUTHENTICATION_FAILED,
"Invalid authentication: it is not allowed "
"to use Authorization HTTP header and authentication via parameters simultaneously");
std::string scheme;
std::string auth_info;
@ -438,7 +429,8 @@ bool HTTPHandler::authenticateUser(
}
else
{
throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: '{}' HTTP Authorization scheme is not supported", scheme);
throw Exception(
ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: '{}' HTTP Authorization scheme is not supported", scheme);
}
}
else
@ -464,7 +456,8 @@ bool HTTPHandler::authenticateUser(
auto * gss_acceptor_context = dynamic_cast<GSSAcceptorContext *>(request_credentials.get());
if (!gss_acceptor_context)
throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: unexpected 'Negotiate' HTTP Authorization scheme expected");
throw Exception(
ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: unexpected 'Negotiate' HTTP Authorization scheme expected");
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wunreachable-code"
@ -500,9 +493,10 @@ bool HTTPHandler::authenticateUser(
if (params.has("quota_key"))
{
if (!quota_key.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Invalid authentication: it is not allowed "
"to use quota key as HTTP header and as parameter simultaneously");
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Invalid authentication: it is not allowed "
"to use quota key as HTTP header and as parameter simultaneously");
quota_key = params.get("quota_key");
}
@ -627,26 +621,29 @@ void HTTPHandler::processQuery(
size_t buffer_size_memory = (buffer_size_total > buffer_size_http) ? buffer_size_total : 0;
bool enable_http_compression = params.getParsed<bool>("enable_http_compression", context->getSettingsRef().enable_http_compression);
Int64 http_zlib_compression_level = params.getParsed<Int64>("http_zlib_compression_level", context->getSettingsRef().http_zlib_compression_level);
Int64 http_zlib_compression_level
= params.getParsed<Int64>("http_zlib_compression_level", context->getSettingsRef().http_zlib_compression_level);
used_output.out_holder =
std::make_shared<WriteBufferFromHTTPServerResponse>(
response,
request.getMethod() == HTTPRequest::HTTP_HEAD,
context->getServerSettings().keep_alive_timeout.totalSeconds(),
write_event);
used_output.out_holder = std::make_shared<WriteBufferFromHTTPServerResponse>(
response,
request.getMethod() == HTTPRequest::HTTP_HEAD,
context->getServerSettings().keep_alive_timeout.totalSeconds(),
write_event);
used_output.out = used_output.out_holder;
used_output.out_maybe_compressed = used_output.out_holder;
if (client_supports_http_compression && enable_http_compression)
{
used_output.out_holder->setCompressionMethodHeader(http_response_compression_method);
used_output.wrap_compressed_holder =
wrapWriteBufferWithCompressionMethod(
used_output.out_holder.get(),
http_response_compression_method,
static_cast<int>(http_zlib_compression_level),
0, DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0, false);
used_output.wrap_compressed_holder = wrapWriteBufferWithCompressionMethod(
used_output.out_holder.get(),
http_response_compression_method,
static_cast<int>(http_zlib_compression_level),
0,
DBMS_DEFAULT_BUFFER_SIZE,
nullptr,
0,
false);
used_output.out = used_output.wrap_compressed_holder;
}
@ -670,16 +667,13 @@ void HTTPHandler::processQuery(
{
auto tmp_data = std::make_shared<TemporaryDataOnDisk>(server.context()->getTempDataOnDisk());
auto create_tmp_disk_buffer = [tmp_data] (const WriteBufferPtr &) -> WriteBufferPtr
{
return tmp_data->createRawStream();
};
auto create_tmp_disk_buffer = [tmp_data](const WriteBufferPtr &) -> WriteBufferPtr { return tmp_data->createRawStream(); };
cascade_buffer2.emplace_back(std::move(create_tmp_disk_buffer));
}
else
{
auto push_memory_buffer_and_continue = [next_buffer = used_output.out_maybe_compressed] (const WriteBufferPtr & prev_buf)
auto push_memory_buffer_and_continue = [next_buffer = used_output.out_maybe_compressed](const WriteBufferPtr & prev_buf)
{
auto * prev_memory_buffer = typeid_cast<MemoryWriteBuffer *>(prev_buf.get());
if (!prev_memory_buffer)
@ -694,7 +688,8 @@ void HTTPHandler::processQuery(
cascade_buffer2.emplace_back(push_memory_buffer_and_continue);
}
used_output.out_delayed_and_compressed_holder = std::make_unique<CascadeWriteBuffer>(std::move(cascade_buffer1), std::move(cascade_buffer2));
used_output.out_delayed_and_compressed_holder
= std::make_unique<CascadeWriteBuffer>(std::move(cascade_buffer1), std::move(cascade_buffer2));
used_output.out_maybe_delayed_and_compressed = used_output.out_delayed_and_compressed_holder.get();
}
else
@ -707,7 +702,8 @@ void HTTPHandler::processQuery(
int zstd_window_log_max = static_cast<int>(context->getSettingsRef().zstd_window_log_max);
auto in_post = wrapReadBufferWithCompressionMethod(
wrapReadBufferReference(request.getStream()),
chooseCompressionMethod({}, http_request_compression_method_str), zstd_window_log_max);
chooseCompressionMethod({}, http_request_compression_method_str),
zstd_window_log_max);
/// The data can also be compressed using incompatible internal algorithm. This is indicated by
/// 'decompress' query parameter.
@ -723,12 +719,26 @@ void HTTPHandler::processQuery(
std::unique_ptr<ReadBuffer> in;
static const NameSet reserved_param_names{"compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", "role",
"buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check", "client_protocol_version", "close_session"};
static const NameSet reserved_param_names{
"compress",
"decompress",
"user",
"password",
"quota_key",
"query_id",
"stacktrace",
"role",
"buffer_size",
"wait_end_of_query",
"session_id",
"session_timeout",
"session_check",
"client_protocol_version",
"close_session"};
Names reserved_param_suffixes;
auto param_could_be_skipped = [&] (const String & name)
auto param_could_be_skipped = [&](const String & name)
{
/// Empty parameter appears when URL like ?&a=b or a=b&&c=d. Just skip them for user's convenience.
if (name.empty())
@ -738,10 +748,8 @@ void HTTPHandler::processQuery(
return true;
for (const String & suffix : reserved_param_suffixes)
{
if (endsWith(name, suffix))
return true;
}
return false;
};
@ -859,47 +867,47 @@ void HTTPHandler::processQuery(
if (settings.add_http_cors_header && !request.get("Origin", "").empty() && !config.has("http_options_response"))
used_output.out_holder->addHeaderCORS(true);
auto append_callback = [my_context = context] (ProgressCallback callback)
auto append_callback = [my_context = context](ProgressCallback callback)
{
auto prev = my_context->getProgressCallback();
my_context->setProgressCallback([prev, callback] (const Progress & progress)
{
if (prev)
prev(progress);
my_context->setProgressCallback(
[prev, callback](const Progress & progress)
{
if (prev)
prev(progress);
callback(progress);
});
callback(progress);
});
};
/// While still no data has been sent, we will report about query execution progress by sending HTTP headers.
/// Note that we add it unconditionally so the progress is available for `X-ClickHouse-Summary`
append_callback([&used_output](const Progress & progress)
{
used_output.out_holder->onProgress(progress);
});
append_callback([&used_output](const Progress & progress) { used_output.out_holder->onProgress(progress); });
if (settings.readonly > 0 && settings.cancel_http_readonly_queries_on_client_close)
{
append_callback([&context, &request](const Progress &)
{
/// Assume that at the point this method is called no one is reading data from the socket any more:
/// should be true for read-only queries.
if (!request.checkPeerConnected())
context->killCurrentQuery();
});
append_callback(
[&context, &request](const Progress &)
{
/// Assume that at the point this method is called no one is reading data from the socket any more:
/// should be true for read-only queries.
if (!request.checkPeerConnected())
context->killCurrentQuery();
});
}
customizeContext(request, context, *in_post_maybe_compressed);
in = has_external_data ? std::move(in_param) : std::make_unique<ConcatReadBuffer>(*in_param, *in_post_maybe_compressed);
auto set_query_result = [&response, this] (const QueryResultDetails & details)
auto set_query_result = [&response, this](const QueryResultDetails & details)
{
response.add("X-ClickHouse-Query-Id", details.query_id);
if (http_response_headers_override)
for (auto [header_name, header_value] : *http_response_headers_override)
response.add(header_name, header_value);
if (content_type_override)
response.setContentType(*content_type_override);
else if (details.content_type)
if (response.getContentType() == Poco::Net::HTTPMessage::UNKNOWN_CONTENT_TYPE && details.content_type)
response.setContentType(*details.content_type);
if (details.format)
@ -909,7 +917,10 @@ void HTTPHandler::processQuery(
response.add("X-ClickHouse-Timezone", *details.timezone);
};
auto handle_exception_in_output_format = [&](IOutputFormat & current_output_format, const String & format_name, const ContextPtr & context_, const std::optional<FormatSettings> & format_settings)
auto handle_exception_in_output_format = [&](IOutputFormat & current_output_format,
const String & format_name,
const ContextPtr & context_,
const std::optional<FormatSettings> & format_settings)
{
if (settings.http_write_exception_in_output_format && current_output_format.supportsWritingException())
{
@ -929,7 +940,8 @@ void HTTPHandler::processQuery(
}
else
{
bool with_stacktrace = (params.getParsed<bool>("stacktrace", false) && server.config().getBool("enable_http_stacktrace", true));
bool with_stacktrace
= (params.getParsed<bool>("stacktrace", false) && server.config().getBool("enable_http_stacktrace", true));
ExecutionStatus status = ExecutionStatus::fromCurrentException("", with_stacktrace);
formatExceptionForClient(status.code, request, response, used_output);
current_output_format.setException(status.message);
@ -970,7 +982,8 @@ try
if (!used_output.out_holder && !used_output.exception_is_written)
{
/// If nothing was sent yet and we don't even know if we must compress the response.
WriteBufferFromHTTPServerResponse(response, request.getMethod() == HTTPRequest::HTTP_HEAD, DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT).writeln(s);
WriteBufferFromHTTPServerResponse(response, request.getMethod() == HTTPRequest::HTTP_HEAD, DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT)
.writeln(s);
}
else if (used_output.out_maybe_compressed)
{
@ -1034,7 +1047,8 @@ catch (...)
}
}
void HTTPHandler::formatExceptionForClient(int exception_code, HTTPServerRequest & request, HTTPServerResponse & response, Output & used_output)
void HTTPHandler::formatExceptionForClient(
int exception_code, HTTPServerRequest & request, HTTPServerResponse & response, Output & used_output)
{
if (used_output.out_holder)
used_output.out_holder->setExceptionCode(exception_code);
@ -1101,9 +1115,7 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
std::string opentelemetry_traceparent = request.get("traceparent");
std::string error;
if (!client_trace_context.parseTraceparentHeader(opentelemetry_traceparent, error))
{
LOG_DEBUG(log, "Failed to parse OpenTelemetry traceparent header '{}': {}", opentelemetry_traceparent, error);
}
client_trace_context.tracestate = request.get("tracestate", "");
}
@ -1141,9 +1153,10 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
/// Workaround. Poco does not detect 411 Length Required case.
if (request.getMethod() == HTTPRequest::HTTP_POST && !request.getChunkedTransferEncoding() && !request.hasContentLength())
{
throw Exception(ErrorCodes::HTTP_LENGTH_REQUIRED,
"The Transfer-Encoding is not chunked and there "
"is no Content-Length header for POST request");
throw Exception(
ErrorCodes::HTTP_LENGTH_REQUIRED,
"The Transfer-Encoding is not chunked and there "
"is no Content-Length header for POST request");
}
processQuery(request, params, response, used_output, query_scope, write_event);
@ -1155,7 +1168,8 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
catch (...)
{
SCOPE_EXIT({
request_credentials.reset(); // ...so that the next requests on the connection have to always start afresh in case of exceptions.
request_credentials
.reset(); // ...so that the next requests on the connection have to always start afresh in case of exceptions.
});
/// Check if exception was thrown in used_output.finalize().
@ -1185,15 +1199,18 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
used_output.finalize();
}
DynamicQueryHandler::DynamicQueryHandler(IServer & server_, const std::string & param_name_, const std::optional<String>& content_type_override_)
: HTTPHandler(server_, "DynamicQueryHandler", content_type_override_), param_name(param_name_)
DynamicQueryHandler::DynamicQueryHandler(
IServer & server_,
const std::string & param_name_,
const std::optional<std::unordered_map<String, String>> & http_response_headers_override_)
: HTTPHandler(server_, "DynamicQueryHandler", http_response_headers_override_), param_name(param_name_)
{
}
bool DynamicQueryHandler::customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value)
{
if (key == param_name)
return true; /// do nothing
return true; /// do nothing
if (startsWith(key, QUERY_PARAMETER_NAME_PREFIX))
{
@ -1227,16 +1244,10 @@ std::string DynamicQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm
std::string full_query;
/// Params are of both form params POST and uri (GET params)
for (const auto & it : params)
{
if (it.first == param_name)
{
full_query += it.second;
}
else
{
customizeQueryParam(context, it.first, it.second);
}
}
return full_query;
}
@ -1247,8 +1258,8 @@ PredefinedQueryHandler::PredefinedQueryHandler(
const std::string & predefined_query_,
const CompiledRegexPtr & url_regex_,
const std::unordered_map<String, CompiledRegexPtr> & header_name_with_regex_,
const std::optional<String> & content_type_override_)
: HTTPHandler(server_, "PredefinedQueryHandler", content_type_override_)
const std::optional<std::unordered_map<String, String>> & http_response_headers_override_)
: HTTPHandler(server_, "PredefinedQueryHandler", http_response_headers_override_)
, receive_params(receive_params_)
, predefined_query(predefined_query_)
, url_regex(url_regex_)
@ -1334,20 +1345,37 @@ std::string PredefinedQueryHandler::getQuery(HTTPServerRequest & request, HTMLFo
return predefined_query;
}
HTTPRequestHandlerFactoryPtr createDynamicHandlerFactory(IServer & server,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix)
std::optional<std::unordered_map<String, String>>
parseHttpResponseHeaders(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix)
{
std::unordered_map<String, String> http_response_headers_override;
String http_response_headers_key = config_prefix + ".handler.http_response_headers";
String http_response_headers_key_prefix = http_response_headers_key + ".";
if (config.has(http_response_headers_key))
{
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(config_prefix + ".handler.http_response_headers", keys);
for (const auto & key : keys)
http_response_headers_override[key] = config.getString(http_response_headers_key_prefix + key);
}
if (config.has(config_prefix + ".handler.content_type"))
http_response_headers_override[Poco::Net::HTTPMessage::CONTENT_TYPE] = config.getString(config_prefix + ".handler.content_type");
if (http_response_headers_override.empty())
return std::nullopt;
return std::optional(std::move(http_response_headers_override));
}
HTTPRequestHandlerFactoryPtr
createDynamicHandlerFactory(IServer & server, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix)
{
auto query_param_name = config.getString(config_prefix + ".handler.query_param_name", "query");
std::optional<String> content_type_override;
if (config.has(config_prefix + ".handler.content_type"))
content_type_override = config.getString(config_prefix + ".handler.content_type");
std::optional<std::unordered_map<String, String>> http_response_headers_override = parseHttpResponseHeaders(config, config_prefix);
auto creator = [&server, query_param_name, content_type_override] () -> std::unique_ptr<DynamicQueryHandler>
{
return std::make_unique<DynamicQueryHandler>(server, query_param_name, content_type_override);
};
auto creator = [&server, query_param_name, http_response_headers_override]() -> std::unique_ptr<DynamicQueryHandler>
{ return std::make_unique<DynamicQueryHandler>(server, query_param_name, http_response_headers_override); };
auto factory = std::make_shared<HandlingRuleHTTPHandlerFactory<DynamicQueryHandler>>(std::move(creator));
factory->addFiltersFromConfig(config, config_prefix);
@ -1357,11 +1385,14 @@ HTTPRequestHandlerFactoryPtr createDynamicHandlerFactory(IServer & server,
static inline bool capturingNamedQueryParam(NameSet receive_params, const CompiledRegexPtr & compiled_regex)
{
const auto & capturing_names = compiled_regex->NamedCapturingGroups();
return std::count_if(capturing_names.begin(), capturing_names.end(), [&](const auto & iterator)
{
return std::count_if(receive_params.begin(), receive_params.end(),
[&](const auto & param_name) { return param_name == iterator.first; });
});
return std::count_if(
capturing_names.begin(),
capturing_names.end(),
[&](const auto & iterator)
{
return std::count_if(
receive_params.begin(), receive_params.end(), [&](const auto & param_name) { return param_name == iterator.first; });
});
}
static inline CompiledRegexPtr getCompiledRegex(const std::string & expression)
@ -1369,15 +1400,18 @@ static inline CompiledRegexPtr getCompiledRegex(const std::string & expression)
auto compiled_regex = std::make_shared<const re2::RE2>(expression);
if (!compiled_regex->ok())
throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, "Cannot compile re2: {} for http handling rule, error: {}. "
"Look at https://github.com/google/re2/wiki/Syntax for reference.", expression, compiled_regex->error());
throw Exception(
ErrorCodes::CANNOT_COMPILE_REGEXP,
"Cannot compile re2: {} for http handling rule, error: {}. "
"Look at https://github.com/google/re2/wiki/Syntax for reference.",
expression,
compiled_regex->error());
return compiled_regex;
}
HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix)
HTTPRequestHandlerFactoryPtr
createPredefinedHandlerFactory(IServer & server, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix)
{
if (!config.has(config_prefix + ".handler.query"))
throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "There is no path '{}.handler.query' in configuration file.", config_prefix);
@ -1402,9 +1436,7 @@ HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server,
headers_name_with_regex.emplace(std::make_pair(header_name, regex));
}
std::optional<String> content_type_override;
if (config.has(config_prefix + ".handler.content_type"))
content_type_override = config.getString(config_prefix + ".handler.content_type");
std::optional<std::unordered_map<String, String>> http_response_headers_override = parseHttpResponseHeaders(config, config_prefix);
std::shared_ptr<HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>> factory;
@ -1418,18 +1450,12 @@ HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server,
auto regex = getCompiledRegex(url_expression);
if (capturingNamedQueryParam(analyze_receive_params, regex))
{
auto creator = [
&server,
analyze_receive_params,
predefined_query,
regex,
headers_name_with_regex,
content_type_override]
auto creator
= [&server, analyze_receive_params, predefined_query, regex, headers_name_with_regex, http_response_headers_override]
-> std::unique_ptr<PredefinedQueryHandler>
{
return std::make_unique<PredefinedQueryHandler>(
server, analyze_receive_params, predefined_query, regex,
headers_name_with_regex, content_type_override);
server, analyze_receive_params, predefined_query, regex, headers_name_with_regex, http_response_headers_override);
};
factory = std::make_shared<HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>>(std::move(creator));
factory->addFiltersFromConfig(config, config_prefix);
@ -1437,17 +1463,11 @@ HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server,
}
}
auto creator = [
&server,
analyze_receive_params,
predefined_query,
headers_name_with_regex,
content_type_override]
auto creator = [&server, analyze_receive_params, predefined_query, headers_name_with_regex, http_response_headers_override]
-> std::unique_ptr<PredefinedQueryHandler>
{
return std::make_unique<PredefinedQueryHandler>(
server, analyze_receive_params, predefined_query, CompiledRegexPtr{},
headers_name_with_regex, content_type_override);
server, analyze_receive_params, predefined_query, CompiledRegexPtr{}, headers_name_with_regex, http_response_headers_override);
};
factory = std::make_shared<HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>>(std::move(creator));

View File

@ -1,21 +1,27 @@
#pragma once
#include <optional>
#include <string>
#include <unordered_map>
#include <Compression/CompressedWriteBuffer.h>
#include <Core/Names.h>
#include <IO/CascadeWriteBuffer.h>
#include <Server/HTTP/HTMLForm.h>
#include <Server/HTTP/HTTPRequestHandler.h>
#include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>
#include <Common/CurrentMetrics.h>
#include <Common/CurrentThread.h>
#include <IO/CascadeWriteBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <Common/re2.h>
namespace CurrentMetrics
{
extern const Metric HTTPConnection;
extern const Metric HTTPConnection;
}
namespace Poco { class Logger; }
namespace Poco
{
class Logger;
}
namespace DB
{
@ -31,13 +37,16 @@ using CompiledRegexPtr = std::shared_ptr<const re2::RE2>;
class HTTPHandler : public HTTPRequestHandler
{
public:
HTTPHandler(IServer & server_, const std::string & name, const std::optional<String> & content_type_override_);
HTTPHandler(
IServer & server_,
const std::string & name,
const std::optional<std::unordered_map<String, String>> & http_response_headers_override_);
~HTTPHandler() override;
void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event) override;
/// This method is called right before the query execution.
virtual void customizeContext(HTTPServerRequest & /* request */, ContextMutablePtr /* context */, ReadBuffer & /* body */) {}
virtual void customizeContext(HTTPServerRequest & /* request */, ContextMutablePtr /* context */, ReadBuffer & /* body */) { }
virtual bool customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) = 0;
@ -113,8 +122,8 @@ private:
/// See settings http_max_fields, http_max_field_name_size, http_max_field_value_size in HTMLForm.
const Settings & default_settings;
/// Overrides Content-Type provided by the format of the response.
std::optional<String> content_type_override;
/// Overrides for response headers.
std::optional<std::unordered_map<String, String>> http_response_headers_override;
// session is reset at the end of each request/response.
std::unique_ptr<Session> session;
@ -128,10 +137,7 @@ private:
// Returns false when the user is not authenticated yet, and the 'Negotiate' response is sent,
// the session and request_credentials instances are preserved.
// Throws an exception if authentication failed.
bool authenticateUser(
HTTPServerRequest & request,
HTMLForm & params,
HTTPServerResponse & response);
bool authenticateUser(HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response);
/// Also initializes 'used_output'.
void processQuery(
@ -143,17 +149,9 @@ private:
const ProfileEvents::Event & write_event);
void trySendExceptionToClient(
const std::string & s,
int exception_code,
HTTPServerRequest & request,
HTTPServerResponse & response,
Output & used_output);
const std::string & s, int exception_code, HTTPServerRequest & request, HTTPServerResponse & response, Output & used_output);
void formatExceptionForClient(
int exception_code,
HTTPServerRequest & request,
HTTPServerResponse & response,
Output & used_output);
void formatExceptionForClient(int exception_code, HTTPServerRequest & request, HTTPServerResponse & response, Output & used_output);
static void pushDelayedResults(Output & used_output);
};
@ -162,12 +160,16 @@ class DynamicQueryHandler : public HTTPHandler
{
private:
std::string param_name;
public:
explicit DynamicQueryHandler(IServer & server_, const std::string & param_name_ = "query", const std::optional<String>& content_type_override_ = std::nullopt);
explicit DynamicQueryHandler(
IServer & server_,
const std::string & param_name_ = "query",
const std::optional<std::unordered_map<String, String>> & http_response_headers_override_ = std::nullopt);
std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) override;
bool customizeQueryParam(ContextMutablePtr context, const std::string &key, const std::string &value) override;
bool customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) override;
};
class PredefinedQueryHandler : public HTTPHandler
@ -177,11 +179,15 @@ private:
std::string predefined_query;
CompiledRegexPtr url_regex;
std::unordered_map<String, CompiledRegexPtr> header_name_with_capture_regex;
public:
PredefinedQueryHandler(
IServer & server_, const NameSet & receive_params_, const std::string & predefined_query_
, const CompiledRegexPtr & url_regex_, const std::unordered_map<String, CompiledRegexPtr> & header_name_with_regex_
, const std::optional<std::string> & content_type_override_);
IServer & server_,
const NameSet & receive_params_,
const std::string & predefined_query_,
const CompiledRegexPtr & url_regex_,
const std::unordered_map<String, CompiledRegexPtr> & header_name_with_regex_,
const std::optional<std::unordered_map<String, String>> & http_response_headers_override_ = std::nullopt);
void customizeContext(HTTPServerRequest & request, ContextMutablePtr context, ReadBuffer & body) override;